feat: support to mute by resource classpath_prefix (#785)

This commit is contained in:
Istil 2021-08-27 18:21:36 +08:00 committed by GitHub
parent 351dee6a12
commit ae0b036ae4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 49 additions and 34 deletions

View File

@ -1,6 +1,8 @@
package alert package alert
import ( import (
"strings"
"github.com/didi/nightingale/v5/cache" "github.com/didi/nightingale/v5/cache"
"github.com/didi/nightingale/v5/models" "github.com/didi/nightingale/v5/models"
"github.com/toolkits/pkg/logger" "github.com/toolkits/pkg/logger"
@ -14,13 +16,13 @@ func isEventMute(event *models.AlertEvent) bool {
} }
// 先去匹配一下metric为空的mute // 先去匹配一下metric为空的mute
if matchMute("", event.ResIdent, event.TagMap) { if matchMute("", event.ResIdent, event.TagMap, event.ResClasspaths) {
return true return true
} }
// 如果是与条件就会有多个metric任一个匹配了屏蔽规则都算被屏蔽 // 如果是与条件就会有多个metric任一个匹配了屏蔽规则都算被屏蔽
for i := 0; i < len(historyPoints); i++ { for i := 0; i < len(historyPoints); i++ {
if matchMute(historyPoints[i].Metric, event.ResIdent, event.TagMap) { if matchMute(historyPoints[i].Metric, event.ResIdent, event.TagMap, event.ResClasspaths) {
return true return true
} }
} }
@ -35,7 +37,7 @@ func isEventMute(event *models.AlertEvent) bool {
return false return false
} }
func matchMute(metric, ident string, tags map[string]string) bool { func matchMute(metric, ident string, tags map[string]string, classpaths string) bool {
filters, exists := cache.AlertMute.GetByKey(metric) filters, exists := cache.AlertMute.GetByKey(metric)
if !exists { if !exists {
// 没有屏蔽规则跟这个事件相关 // 没有屏蔽规则跟这个事件相关
@ -44,7 +46,7 @@ func matchMute(metric, ident string, tags map[string]string) bool {
// 只要有一个屏蔽规则命中,那这个事件就是被屏蔽了 // 只要有一个屏蔽规则命中,那这个事件就是被屏蔽了
for _, filter := range filters { for _, filter := range filters {
if matchMuteOnce(filter, ident, tags) { if matchMuteOnce(filter, ident, tags, classpaths) {
return true return true
} }
} }
@ -52,7 +54,15 @@ func matchMute(metric, ident string, tags map[string]string) bool {
return false return false
} }
func matchMuteOnce(filter cache.Filter, ident string, tags map[string]string) bool { func matchMuteOnce(filter cache.Filter, ident string, tags map[string]string, classpaths string) bool {
if len(filter.ClasspathPrefix) != 0 && !strings.HasPrefix(classpaths, filter.ClasspathPrefix) && !strings.Contains(classpaths, " "+filter.ClasspathPrefix) {
//没配置分组屏蔽就不做后续比较
//比如事件的资源calsspath为“n9e.mon n9e.rdb ccp.web”配置屏蔽为n9e.rdb
//只要字符串前缀为n9e.rdb或者字符串包含“ n9e.rdb”即可判断所有alsspath中是否有前缀为n9e.rdb的
//只要有任一点不满足,那这个屏蔽规则也没有继续验证下去的必要
return false
}
if filter.ResReg != nil && !filter.ResReg.MatchString(ident) { if filter.ResReg != nil && !filter.ResReg.MatchString(ident) {
// 比如屏蔽规则配置的是c3-ceph.* // 比如屏蔽规则配置的是c3-ceph.*
// 当前事件的资源标识是c4-ceph01.bj // 当前事件的资源标识是c4-ceph01.bj

5
cache/alert_mute.go vendored
View File

@ -10,8 +10,9 @@ type AlertMuteMap struct {
Data map[string][]Filter Data map[string][]Filter
} }
type Filter struct { type Filter struct {
ResReg *regexp.Regexp ClasspathPrefix string
TagsMap map[string]string ResReg *regexp.Regexp
TagsMap map[string]string
} }
var AlertMute = &AlertMuteMap{Data: make(map[string][]Filter)} var AlertMute = &AlertMuteMap{Data: make(map[string][]Filter)}

View File

@ -23,12 +23,13 @@ func muteGets(c *gin.Context) {
} }
type muteForm struct { type muteForm struct {
Metric string `json:"metric"` ClasspathPrefix string `json:"classpath_prefix "`
ResFilters string `json:"res_filters"` Metric string `json:"metric"`
TagFilters string `json:"tags_filters"` ResFilters string `json:"res_filters"`
Cause string `json:"cause"` TagFilters string `json:"tags_filters"`
Btime int64 `json:"btime"` Cause string `json:"cause"`
Etime int64 `json:"etime"` Btime int64 `json:"btime"`
Etime int64 `json:"etime"`
} }
func muteAdd(c *gin.Context) { func muteAdd(c *gin.Context) {
@ -38,13 +39,14 @@ func muteAdd(c *gin.Context) {
me := loginUser(c).MustPerm("mute_create") me := loginUser(c).MustPerm("mute_create")
mt := models.Mute{ mt := models.Mute{
Metric: f.Metric, ClasspathPrefix: f.ClasspathPrefix,
ResFilters: f.ResFilters, Metric: f.Metric,
TagFilters: f.TagFilters, ResFilters: f.ResFilters,
Cause: f.Cause, TagFilters: f.TagFilters,
Btime: f.Btime, Cause: f.Cause,
Etime: f.Etime, Btime: f.Btime,
CreateBy: me.Username, Etime: f.Etime,
CreateBy: me.Username,
} }
renderMessage(c, mt.Add()) renderMessage(c, mt.Add())

View File

@ -10,18 +10,18 @@ import (
) )
type Mute struct { type Mute struct {
Id int64 `json:"id"` Id int64 `json:"id"`
Metric string `json:"metric"` ClasspathPrefix string `json:"classpath_prefix"`
ResFilters string `json:"res_filters"` Metric string `json:"metric"`
TagFilters string `json:"tags_filters"` ResFilters string `json:"res_filters"`
Cause string `json:"cause"` TagFilters string `json:"tags_filters"`
Btime int64 `json:"btime"` Cause string `json:"cause"`
Etime int64 `json:"etime"` Btime int64 `json:"btime"`
CreateBy string `json:"create_by"` Etime int64 `json:"etime"`
CreateAt int64 `json:"create_at"` CreateBy string `json:"create_by"`
CreateAt int64 `json:"create_at"`
ResRegexp *regexp.Regexp `xorm:"-" json:"-"` ResRegexp *regexp.Regexp `xorm:"-" json:"-"`
TagsMap map[string]string `xorm:"-" json:"-"` TagsMap map[string]string `xorm:"-" json:"-"`
} }
func (m *Mute) TableName() string { func (m *Mute) TableName() string {

View File

@ -160,6 +160,7 @@ CREATE TABLE `classpath_favorite` (
CREATE TABLE `mute` ( CREATE TABLE `mute` (
`id` bigint unsigned not null auto_increment, `id` bigint unsigned not null auto_increment,
`classpath_prefix` varchar(255) not null default '' comment 'classpath prefix',
`metric` varchar(255) not null comment 'required', `metric` varchar(255) not null comment 'required',
`res_filters` varchar(4096) not null default 'resource filters', `res_filters` varchar(4096) not null default 'resource filters',
`tag_filters` varchar(8192) not null default '', `tag_filters` varchar(8192) not null default '',

View File

@ -58,8 +58,9 @@ func syncAlertMutes() error {
} }
filter := cache.Filter{ filter := cache.Filter{
ResReg: mutes[i].ResRegexp, ResReg: mutes[i].ResRegexp,
TagsMap: mutes[i].TagsMap, TagsMap: mutes[i].TagsMap,
ClasspathPrefix: mutes[i].ClasspathPrefix,
} }
muteMap[mutes[i].Metric] = append(muteMap[mutes[i].Metric], filter) muteMap[mutes[i].Metric] = append(muteMap[mutes[i].Metric], filter)