From 0b4e3b96563f80d3ca138a9be5a5bc5e355a3d62 Mon Sep 17 00:00:00 2001 From: Ulric Qin Date: Sat, 14 Aug 2021 23:17:06 +0800 Subject: [PATCH] fix judge prom --- judge/handler.go | 10 +++--- judge/last_event.go | 77 ++++++++++++++++++++++++++++++--------------- judge/prome_pull.go | 4 +-- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/judge/handler.go b/judge/handler.go index 265aee70..0a68b315 100644 --- a/judge/handler.go +++ b/judge/handler.go @@ -449,14 +449,14 @@ func sendEventIfNeed(status []bool, event *models.AlertEvent, stra *models.Alert } now := time.Now().Unix() - lastEvent, exists := LastEvents.Get(event.HashId) + lastEvent, exists := LastEvents.Get(event.RuleId, event.HashId) switch event.IsPromePull { case 0: // push型的 && 与条件型的 if exists && lastEvent.IsPromePull == 1 { // 之前内存中的事件是pull型的，先清空内存中的事件 - LastEvents.Del(event.HashId) + LastEvents.Del(event.RuleId, event.HashId) } if isTriggered { @@ -476,7 +476,7 @@ func sendEventIfNeed(status []bool, event *models.AlertEvent, stra *models.Alert // pull型的，产生的事件一定是触发了阈值的，即这个case里不存在recovery的场景，recovery的场景用resolve_timeout的cron来处理 if exists && lastEvent.IsPromePull == 0 { // 之前内存中的事件是push型的，先清空内存中的事件 - LastEvents.Del(event.HashId) + LastEvents.Del(event.RuleId, event.HashId) } // 1. 第一次来，并且AlertDuration=0，直接发送 @@ -490,7 +490,7 @@ func sendEventIfNeed(status []bool, event *models.AlertEvent, stra *models.Alert SendEvent(event) } else { // 只有一条事件，显然无法满足for AlertDuration的时间，放到内存里等待 - LastEvents.Set(event.HashId, event) + LastEvents.Set(event) } return } @@ -529,7 +529,7 @@ func sendEventIfNeed(status []bool, event *models.AlertEvent, stra *models.Alert func SendEvent(event *models.AlertEvent) { // update last event - LastEvents.Set(event.HashId, event) + LastEvents.Set(event) ok := EventQueue.PushFront(event) if !ok { logger.Errorf("push event:%v err", event) diff --git a/judge/last_event.go b/judge/last_event.go index 04462450..3333fb58 100644 --- a/judge/last_event.go +++ b/judge/last_event.go @@ -4,58 +4,85 @@ import ( "sync" "time" - "github.com/toolkits/pkg/logger" - "github.com/didi/nightingale/v5/models" + "github.com/toolkits/pkg/logger" ) +// rule_id -> hash_id -> *models.AlertEvent type SafeEventMap struct { sync.RWMutex - M map[string]*models.AlertEvent + M map[int64]map[string]*models.AlertEvent } var ( - LastEvents = &SafeEventMap{M: make(map[string]*models.AlertEvent)} + LastEvents = &SafeEventMap{M: make(map[int64]map[string]*models.AlertEvent)} ) -func (s *SafeEventMap) Get(key string) (*models.AlertEvent, bool) { +func (s *SafeEventMap) Get(ruleId int64, hashId string) (*models.AlertEvent, bool) { s.RLock() defer s.RUnlock() - event, exists := s.M[key] - return event, exists + + m, has := s.M[ruleId] + if !has { + return nil, false + } + + event, has := m[hashId] + return event, has } -func (s *SafeEventMap) Set(key string, event *models.AlertEvent) { +func (s *SafeEventMap) Set(event *models.AlertEvent) { s.Lock() defer s.Unlock() - s.M[key] = event + + m, has := s.M[event.RuleId] + if !has { + m = make(map[string]*models.AlertEvent) + m[event.HashId] = event + s.M[event.RuleId] = m + } else { + s.M[event.RuleId][event.HashId] = event + } } -func (s *SafeEventMap) Del(key string) { +func (s *SafeEventMap) Del(ruleId int64, hashId string) { s.Lock() defer s.Unlock() - delete(s.M, key) + + _, has := s.M[ruleId] + if !has { + return + } + + delete(s.M[ruleId], hashId) } -func (s *SafeEventMap) DeleteOrSendRecovery(promql string, toKeepKeys map[string]struct{}) { +func (s *SafeEventMap) DeleteOrSendRecovery(ruleId int64, toKeepKeys map[string]struct{}) { s.Lock() defer s.Unlock() - for k, ev := range s.M { + + m, has := s.M[ruleId] + if !has { + return + } + + for k, ev := range m { if _, loaded := toKeepKeys[k]; loaded { continue } - if ev.ReadableExpression == promql { - logger.Debugf("[to_del][ev.IsRecovery:%+v][ev.LastSend:%+v][promql:%v]", ev.IsRecovery, ev.LastSend, promql) - now := time.Now().Unix() - // promql 没查询到结果，需要将告警标记为已恢复并发送 - // 同时需要满足 已经发送过触发信息，并且时间差满足 大于AlertDuration - // 为了避免 发送告警后 一个点 断点了就立即发送恢复信息的case - if ev.IsAlert() && ev.LastSend && now-ev.TriggerTime > ev.AlertDuration { - logger.Debugf("[prom.alert.MarkRecov][promql:%v][ev.RuleName:%v]", promql, ev.RuleName) - ev.MarkRecov() - EventQueue.PushFront(ev) - delete(s.M, k) - } + + // 如果因为promql修改，导致本来是告警状态变成了恢复，也接受 + logger.Debugf("[to_del][ev.IsRecovery:%+v][ev.LastSend:%+v]", ev.IsRecovery, ev.LastSend) + + // promql 没查询到结果，需要将告警标记为已恢复并发送 + // 同时需要满足 已经发送过触发信息，并且时间差满足 大于AlertDuration + // 为了避免 发送告警后 一个点 断点了就立即发送恢复信息的case + now := time.Now().Unix() + if ev.IsAlert() && ev.LastSend && now-ev.TriggerTime > ev.AlertDuration { + logger.Debugf("[prom.alert.MarkRecov][ev.RuleName:%v]", ev.RuleName) + ev.MarkRecov() + EventQueue.PushFront(ev) + delete(s.M[ruleId], k) } } } diff --git a/judge/prome_pull.go b/judge/prome_pull.go index 45c58787..a5d09e63 100644 --- a/judge/prome_pull.go +++ b/judge/prome_pull.go @@ -121,7 +121,7 @@ func handlePromqlVector(pv promql.Vector, r models.AlertRule) { toKeepKeys := map[string]struct{}{} if len(pv) == 0 { // 说明没触发，或者没查询到，删掉rule-id开头的所有event - LastEvents.DeleteOrSendRecovery(r.PullExpr.PromQl, toKeepKeys) + LastEvents.DeleteOrSendRecovery(r.Id, toKeepKeys) return } @@ -191,6 +191,6 @@ func handlePromqlVector(pv promql.Vector, r models.AlertRule) { logger.Debugf("[handlePromqlVector_has_value][event:%+v]\n", event) sendEventIfNeed([]bool{true}, event, &r) } - LastEvents.DeleteOrSendRecovery(r.PullExpr.PromQl, toKeepKeys) + LastEvents.DeleteOrSendRecovery(r.Id, toKeepKeys) }