89 lines
1.9 KiB
Go
89 lines
1.9 KiB
Go
package judge
|
||
|
||
import (
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/didi/nightingale/v5/models"
|
||
"github.com/toolkits/pkg/logger"
|
||
)
|
||
|
||
// rule_id -> hash_id -> *models.AlertEvent
|
||
type SafeEventMap struct {
|
||
sync.RWMutex
|
||
M map[int64]map[string]*models.AlertEvent
|
||
}
|
||
|
||
var (
|
||
LastEvents = &SafeEventMap{M: make(map[int64]map[string]*models.AlertEvent)}
|
||
)
|
||
|
||
func (s *SafeEventMap) Get(ruleId int64, hashId string) (*models.AlertEvent, bool) {
|
||
s.RLock()
|
||
defer s.RUnlock()
|
||
|
||
m, has := s.M[ruleId]
|
||
if !has {
|
||
return nil, false
|
||
}
|
||
|
||
event, has := m[hashId]
|
||
return event, has
|
||
}
|
||
|
||
func (s *SafeEventMap) Set(event *models.AlertEvent) {
|
||
s.Lock()
|
||
defer s.Unlock()
|
||
|
||
m, has := s.M[event.RuleId]
|
||
if !has {
|
||
m = make(map[string]*models.AlertEvent)
|
||
m[event.HashId] = event
|
||
s.M[event.RuleId] = m
|
||
} else {
|
||
s.M[event.RuleId][event.HashId] = event
|
||
}
|
||
}
|
||
|
||
func (s *SafeEventMap) Del(ruleId int64, hashId string) {
|
||
s.Lock()
|
||
defer s.Unlock()
|
||
|
||
_, has := s.M[ruleId]
|
||
if !has {
|
||
return
|
||
}
|
||
|
||
delete(s.M[ruleId], hashId)
|
||
}
|
||
|
||
func (s *SafeEventMap) DeleteOrSendRecovery(ruleId int64, toKeepKeys map[string]struct{}) {
|
||
s.Lock()
|
||
defer s.Unlock()
|
||
|
||
m, has := s.M[ruleId]
|
||
if !has {
|
||
return
|
||
}
|
||
|
||
for k, ev := range m {
|
||
if _, loaded := toKeepKeys[k]; loaded {
|
||
continue
|
||
}
|
||
|
||
// 如果因为promql修改，导致本来是告警状态变成了恢复，也接受
|
||
logger.Debugf("[to_del][ev.IsRecovery:%+v][ev.LastSend:%+v]", ev.IsRecovery, ev.LastSend)
|
||
|
||
// promql 没查询到结果，需要将告警标记为已恢复并发送
|
||
// 同时需要满足 已经发送过触发信息，并且时间差满足 大于AlertDuration
|
||
// 为了避免 发送告警后 一个点 断点了就立即发送恢复信息的case
|
||
now := time.Now().Unix()
|
||
if ev.IsAlert() && ev.LastSend && now-ev.TriggerTime > ev.AlertDuration {
|
||
logger.Debugf("[prom.alert.MarkRecov][ev.RuleName:%v]", ev.RuleName)
|
||
ev.MarkRecov()
|
||
EventQueue.PushFront(ev)
|
||
delete(s.M[ruleId], k)
|
||
}
|
||
}
|
||
}
|