diff --git a/src/dataobj/event.go b/src/dataobj/event.go index f62bc87d..157b466b 100644 --- a/src/dataobj/event.go +++ b/src/dataobj/event.go @@ -20,6 +20,24 @@ type History struct { Metric string `json:"metric"` // 指标名 Tags map[string]string `json:"tags,omitempty"` // endpoint/counter Granularity int `json:"-"` // alarm补齐数据时需要 - Points []*RRDData `json:"points"` // 现场值 - Extra string `json:"extra"` + Points []*HistoryData `json:"points"` // 现场值 +} + +type HistoryData struct { + Timestamp int64 `json:"timestamp"` + Value JsonFloat `json:"value"` + Extra string `json:"extra"` +} + +func RRDData2HistoryData(datas []*RRDData) []*HistoryData { + historyDatas := make([]*HistoryData, len(datas)) + + for i := range datas { + historyData := &HistoryData{ + Timestamp: datas[i].Timestamp, + Value: datas[i].Value, + } + historyDatas[i] = historyData + } + return historyDatas } diff --git a/src/dataobj/judge.go b/src/dataobj/judge.go index 5031084b..966c9271 100644 --- a/src/dataobj/judge.go +++ b/src/dataobj/judge.go @@ -28,9 +28,3 @@ func (j *JudgeItem) PrimaryKey() string { func (j *JudgeItem) MD5() string { return gstr.MD5(str.PK(strconv.FormatInt(j.Sid, 16), j.Endpoint, j.Metric, str.SortedTags(j.TagsMap))) } - -//告警现场的值 -type HistoryData struct { - Timestamp int64 `json:"timestamp"` - Value float64 `json:"value"` -} diff --git a/src/model/event.go b/src/model/event.go index c7a44acb..7fccc5f6 100644 --- a/src/model/event.go +++ b/src/model/event.go @@ -39,12 +39,12 @@ type EventDetail struct { Tags map[string]string `json:"tags"` Points []*EventDetailPoint `json:"points"` PredPoints []*EventDetailPoint `json:"pred_points,omitempty"` // 预测值, 预测值不为空时, 现场值对应的是实际值 - Extra string `json:"extra"` } type EventDetailPoint struct { Timestamp int64 `json:"timestamp"` Value float64 `json:"value"` + Extra string `json:"extra"` } type EventAlertUpgrade struct { diff --git a/src/model/event_cur.go b/src/model/event_cur.go index f6a2f73f..cd568615 100644 --- a/src/model/event_cur.go +++ b/src/model/event_cur.go @@ -232,7 +232,7 @@ func EventCurGet(col string, value interface{}) (*EventCur, error) { } func (e *EventCur) EventIgnore() error { - _, err := DB["mon"].Exec("update event_cur set ignore_alert=1 where id=?", e.Id) + _, err := DB["mon"].Where("id=?", e.Id).Delete(new(EventCur)) return err } diff --git a/src/modules/judge/cache/index.go b/src/modules/judge/cache/index.go index 0dac52dc..1f38ee80 100644 --- a/src/modules/judge/cache/index.go +++ b/src/modules/judge/cache/index.go @@ -71,7 +71,7 @@ func (i *IndexMap) Clean() { } for key, series := range index { - if now-series.TS > 3600 { + if now-series.TS > 300 { delete(i.Data[id], key) } } diff --git a/src/modules/judge/cache/linkedlist.go b/src/modules/judge/cache/linkedlist.go index eb7a414b..fde1ec55 100644 --- a/src/modules/judge/cache/linkedlist.go +++ b/src/modules/judge/cache/linkedlist.go @@ -54,22 +54,22 @@ func (ll *SafeLinkedList) PushFrontAndMaintain(v *dataobj.JudgeItem, maxCount in // @param limit 至多返回这些，如果不够，有多少返回多少 // @return bool isEnough -func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.RRDData, bool) { +func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.HistoryData, bool) { if limit < 1 { // 其实limit不合法，此处也返回false吧，上层代码要注意 // 因为false通常使上层代码进入异常分支，这样就统一了 - return []*dataobj.RRDData{}, false + return []*dataobj.HistoryData{}, false } size := ll.Len() if size == 0 { - return []*dataobj.RRDData{}, false + return []*dataobj.HistoryData{}, false } firstElement := ll.Front() firstItem := firstElement.Value.(*dataobj.JudgeItem) - var vs []*dataobj.RRDData + var vs []*dataobj.HistoryData isEnough := true judgeType := firstItem.DsType[0] @@ -79,15 +79,21 @@ func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.RRDData, bool) { limit = size isEnough = false } - vs = make([]*dataobj.RRDData, limit) - vs[0] = &dataobj.RRDData{Timestamp: firstItem.Timestamp, Value: dataobj.JsonFloat(firstItem.Value)} + vs = make([]*dataobj.HistoryData, limit) + vs[0] = &dataobj.HistoryData{ + Timestamp: firstItem.Timestamp, + Value: dataobj.JsonFloat(firstItem.Value), + Extra: firstItem.Extra, + } + i := 1 currentElement := firstElement for i < limit { nextElement := currentElement.Next() - vs[i] = &dataobj.RRDData{ + vs[i] = &dataobj.HistoryData{ Timestamp: nextElement.Value.(*dataobj.JudgeItem).Timestamp, Value: dataobj.JsonFloat(nextElement.Value.(*dataobj.JudgeItem).Value), + Extra: nextElement.Value.(*dataobj.JudgeItem).Extra, } i++ currentElement = nextElement diff --git a/src/modules/judge/judge/func.go b/src/modules/judge/judge/func.go index ade11d99..dba78c62 100644 --- a/src/modules/judge/judge/func.go +++ b/src/modules/judge/judge/func.go @@ -8,7 +8,7 @@ import ( ) type Function interface { - Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) + Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) } type MaxFunction struct { @@ -18,7 +18,7 @@ type MaxFunction struct { RightValue float64 } -func (this MaxFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this MaxFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -42,7 +42,7 @@ type MinFunction struct { RightValue float64 } -func (this MinFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this MinFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -66,7 +66,7 @@ type AllFunction struct { RightValue float64 } -func (this AllFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this AllFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -90,7 +90,7 @@ type SumFunction struct { RightValue float64 } -func (this SumFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this SumFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -112,7 +112,7 @@ type AvgFunction struct { RightValue float64 } -func (this AvgFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this AvgFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -135,7 +135,7 @@ type DiffFunction struct { } // 只要有一个点的diff触发阈值，就报警 -func (this DiffFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this DiffFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -163,7 +163,7 @@ type PDiffFunction struct { RightValue float64 } -func (this PDiffFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this PDiffFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -193,7 +193,7 @@ type HappenFunction struct { RightValue float64 } -func (this HappenFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this HappenFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { for n, i := 0, 0; i < len(vs); i++ { if checkIsTriggered(vs[i].Value, this.Operator, this.RightValue) { n++ @@ -211,7 +211,7 @@ type NodataFunction struct { Function } -func (this NodataFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this NodataFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { for _, value := range vs { if !math.IsNaN(float64(value.Value)) { return value.Value, false @@ -228,7 +228,7 @@ type CAvgAbsFunction struct { CompareValue float64 } -func (this CAvgAbsFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this CAvgAbsFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -253,7 +253,7 @@ type CAvgFunction struct { CompareValue float64 } -func (this CAvgFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this CAvgFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -277,7 +277,7 @@ type CAvgRateAbsFunction struct { CompareValue float64 } -func (this CAvgRateAbsFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this CAvgRateAbsFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } @@ -302,7 +302,7 @@ type CAvgRateFunction struct { CompareValue float64 } -func (this CAvgRateFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) { +func (this CAvgRateFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) { if len(vs) < this.Limit { return } diff --git a/src/modules/judge/judge/judge.go b/src/modules/judge/judge/judge.go index ab3623be..358bae54 100644 --- a/src/modules/judge/judge/judge.go +++ b/src/modules/judge/judge/judge.go @@ -62,7 +62,7 @@ func ToJudge(historyMap *cache.JudgeItemMap, key string, val *dataobj.JudgeItem, Judge(stra, stra.Exprs, historyData, val, now, history, "", "", "", []bool{}) } -func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) { +func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.HistoryData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) { stats.Counter.Set("running", 1) if len(exps) < 1 { @@ -86,10 +86,6 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f Granularity: int(firstItem.Step), Points: historyData, } - if len(history) == 0 { - //只有第一个指标是push的模式，可以获取到extra字段 - h.Extra = firstItem.Extra - } history = append(history, h) defer func() { @@ -135,7 +131,7 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f Tags: "", DsType: "GAUGE", } - Judge(stra, exps[1:], []*dataobj.RRDData{}, judgeItem, now, history, info, value, extra, status) + Judge(stra, exps[1:], []*dataobj.HistoryData{}, judgeItem, now, history, info, value, extra, status) return } @@ -143,7 +139,7 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f firstItem.Endpoint = respData[i].Endpoint firstItem.Tags = getTags(respData[i].Counter) firstItem.Step = respData[i].Step - Judge(stra, exps[1:], respData[i].Values, firstItem, now, history, info, value, extra, status) + Judge(stra, exps[1:], dataobj.RRDData2HistoryData(respData[i].Values), firstItem, now, history, info, value, extra, status) } } else { @@ -162,13 +158,13 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f firstItem.Endpoint = respData[i].Endpoint firstItem.Tags = getTags(respData[i].Counter) firstItem.Step = respData[i].Step - Judge(stra, exps[1:], respData[i].Values, firstItem, now, history, info, value, extra, status) + Judge(stra, exps[1:], dataobj.RRDData2HistoryData(respData[i].Values), firstItem, now, history, info, value, extra, status) } } } } -func judgeItemWithStrategy(stra *model.Stra, historyData []*dataobj.RRDData, exp model.Exp, firstItem *dataobj.JudgeItem, now int64) (leftValue dataobj.JsonFloat, isTriggered bool) { +func judgeItemWithStrategy(stra *model.Stra, historyData []*dataobj.HistoryData, exp model.Exp, firstItem *dataobj.JudgeItem, now int64) (leftValue dataobj.JsonFloat, isTriggered bool) { straFunc := exp.Func straParam := []interface{}{} @@ -298,7 +294,7 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([] stats.Counter.Set("get.index", 1) indexsData, err := query.Xclude(req) if err != nil { - return reqs, err + logger.Warning("get index err:", err) } lostSeries := []cache.Series{} @@ -345,6 +341,10 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([] } seriess := cache.SeriesMap.Get(stra.Id) + if len(seriess) == 0 && err != nil { + return reqs, err + } + step := 0 if len(seriess) > 1 { step = seriess[0].Step @@ -389,7 +389,7 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([] return reqs, nil } -func sendEventIfNeed(historyData []*dataobj.RRDData, status []bool, event *dataobj.Event, stra *model.Stra) { +func sendEventIfNeed(historyData []*dataobj.HistoryData, status []bool, event *dataobj.Event, stra *model.Stra) { isTriggered := true for _, s := range status { isTriggered = isTriggered && s diff --git a/src/modules/judge/judge/nodata.go b/src/modules/judge/judge/nodata.go index 31a7d131..630d155f 100644 --- a/src/modules/judge/judge/nodata.go +++ b/src/modules/judge/judge/nodata.go @@ -51,7 +51,7 @@ func nodataJudge() { } nodataJob.Acquire() - go AsyncJudge(nodataJob, stra, stra.Exprs, []*dataobj.RRDData{}, judgeItem, now, []dataobj.History{}, "", "", "", []bool{}) + go AsyncJudge(nodataJob, stra, stra.Exprs, []*dataobj.HistoryData{}, judgeItem, now, []dataobj.History{}, "", "", "", []bool{}) } return } @@ -80,12 +80,12 @@ func nodataJudge() { } nodataJob.Acquire() - go AsyncJudge(nodataJob, stra, stra.Exprs, data.Values, judgeItem, now, []dataobj.History{}, "", "", "", []bool{}) + go AsyncJudge(nodataJob, stra, stra.Exprs, dataobj.RRDData2HistoryData(data.Values), judgeItem, now, []dataobj.History{}, "", "", "", []bool{}) } } } -func AsyncJudge(sema *semaphore.Semaphore, stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) { +func AsyncJudge(sema *semaphore.Semaphore, stra *model.Stra, exps []model.Exp, historyData []*dataobj.HistoryData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) { defer sema.Release() Judge(stra, exps, historyData, firstItem, now, history, info, value, extra, status) } diff --git a/src/modules/monapi/cron/event_consumer.go b/src/modules/monapi/cron/event_consumer.go index b307fed3..a7c771a0 100644 --- a/src/modules/monapi/cron/event_consumer.go +++ b/src/modules/monapi/cron/event_consumer.go @@ -220,10 +220,9 @@ func isInConverge(event *model.Event) bool { return false } -// 三种情况，不需要升级报警 +// 两种情况，不需要升级报警 // 1，认领的报警不需要升级 -// 2，忽略的报警不需要升级 -// 3，屏蔽的报警不需要升级，屏蔽判断在前面已经有了处理，这个方法不用关注 +// 2，屏蔽的报警不需要升级，屏蔽判断在前面已经有了处理，这个方法不用关注 func needUpgrade(event *model.Event) bool { alertUpgradeKey := PrefixAlertUpgrade + fmt.Sprint(event.HashId) eventAlertKey := PrefixAlertTime + fmt.Sprint(event.HashId) @@ -282,11 +281,6 @@ func needUpgrade(event *model.Event) bool { return false } - // 告警已经忽略了 - if eventCur.IgnoreAlert == 1 { - return false - } - // 告警之后，比如30分钟没有处理，就需要升级，那首先得知道首次告警时间 if !redisc.HasKey(eventAlertKey) { err := redisc.SetWithTTL(eventAlertKey, now, 30*24*3600) diff --git a/src/modules/monapi/http/routes/event.go b/src/modules/monapi/http/routes/event.go index 6808f637..3c4d531e 100644 --- a/src/modules/monapi/http/routes/event.go +++ b/src/modules/monapi/http/routes/event.go @@ -9,6 +9,7 @@ import ( "github.com/toolkits/pkg/errors" + "github.com/didi/nightingale/src/dataobj" "github.com/didi/nightingale/src/model" jsoniter "github.com/json-iterator/go" ) @@ -79,9 +80,9 @@ func eventCurGets(c *gin.Context) { err = json.Unmarshal([]byte(events[i].Detail), &detail) errors.Dangerous(err) - tagsList := []string{} - for k, v := range detail[0].Tags { - tagsList = append(tagsList, fmt.Sprintf("%s=%s", k, v)) + var tags string + if len(detail) > 0 { + tags = dataobj.SortedTags(detail[0].Tags) } alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade) @@ -106,7 +107,7 @@ func eventCurGets(c *gin.Context) { Etime: events[i].Etime, Value: events[i].Value, Info: events[i].Info, - Tags: strings.Join(tagsList, ","), + Tags: tags, Created: events[i].Created, Nid: events[i].Nid, Users: users, @@ -164,9 +165,9 @@ func eventHisGets(c *gin.Context) { err = json.Unmarshal([]byte(events[i].Detail), &detail) errors.Dangerous(err) - tagsList := []string{} - for k, v := range detail[0].Tags { - tagsList = append(tagsList, fmt.Sprintf("%s=%s", k, v)) + var tags string + if len(detail) > 0 { + tags = dataobj.SortedTags(detail[0].Tags) } alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade) @@ -191,7 +192,7 @@ func eventHisGets(c *gin.Context) { Etime: events[i].Etime, Value: events[i].Value, Info: events[i].Info, - Tags: strings.Join(tagsList, ","), + Tags: tags, Created: events[i].Created, Nid: events[i].Nid, Users: users,