innov
/
nightingale1
1
0
Fork 0
Code Issues Pull Requests 1 Projects Releases Wiki Activity

refactor: sort event tags & change extra

This commit is contained in:
710leo 2020-04-23 18:30:56 +08:00
parent 322eb97a45
commit d9b89f24b5
11 changed files with 75 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/dataobj/event.go
View File

@ -20,6 +20,24 @@ type History struct {
Metric string `json:"metric"` // 指标名
Tags map[string]string `json:"tags,omitempty"` // endpoint/counter
Granularity int `json:"-"` // alarm补齐数据时需要
Points []*RRDData `json:"points"` // 现场值
Extra string `json:"extra"`
Points []*HistoryData `json:"points"` // 现场值
}
type HistoryData struct {
Timestamp int64 `json:"timestamp"`
Value JsonFloat `json:"value"`
Extra string `json:"extra"`
}
func RRDData2HistoryData(datas []*RRDData) []*HistoryData {
historyDatas := make([]*HistoryData, len(datas))
for i := range datas {
historyData := &HistoryData{
Timestamp: datas[i].Timestamp,
Value: datas[i].Value,
}
historyDatas[i] = historyData
}
return historyDatas
}

6
src/dataobj/judge.go
View File

@ -28,9 +28,3 @@ func (j *JudgeItem) PrimaryKey() string {
func (j *JudgeItem) MD5() string {
return gstr.MD5(str.PK(strconv.FormatInt(j.Sid, 16), j.Endpoint, j.Metric, str.SortedTags(j.TagsMap)))
}
//告警现场的值
type HistoryData struct {
Timestamp int64 `json:"timestamp"`
Value float64 `json:"value"`
}

2
src/model/event.go
View File

@ -39,12 +39,12 @@ type EventDetail struct {
Tags map[string]string `json:"tags"`
Points []*EventDetailPoint `json:"points"`
PredPoints []*EventDetailPoint `json:"pred_points,omitempty"` // 预测值, 预测值不为空时, 现场值对应的是实际值
Extra string `json:"extra"`
}
type EventDetailPoint struct {
Timestamp int64 `json:"timestamp"`
Value float64 `json:"value"`
Extra string `json:"extra"`
}
type EventAlertUpgrade struct {

2
src/model/event_cur.go
View File

@ -232,7 +232,7 @@ func EventCurGet(col string, value interface{}) (*EventCur, error) {
}
func (e *EventCur) EventIgnore() error {
_, err := DB["mon"].Exec("update event_cur set ignore_alert=1 where id=?", e.Id)
_, err := DB["mon"].Where("id=?", e.Id).Delete(new(EventCur))
return err
}

2
src/modules/judge/cache/index.go vendored
View File

@ -71,7 +71,7 @@ func (i *IndexMap) Clean() {
}
for key, series := range index {
if now-series.TS > 3600 {
if now-series.TS > 300 {
delete(i.Data[id], key)
}
}

20
src/modules/judge/cache/linkedlist.go vendored
View File

@ -54,22 +54,22 @@ func (ll *SafeLinkedList) PushFrontAndMaintain(v *dataobj.JudgeItem, maxCount in
// @param limit 至多返回这些，如果不够，有多少返回多少
// @return bool isEnough
func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.RRDData, bool) {
func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.HistoryData, bool) {
if limit < 1 {
// 其实limit不合法，此处也返回false吧，上层代码要注意
// 因为false通常使上层代码进入异常分支，这样就统一了
return []*dataobj.RRDData{}, false
return []*dataobj.HistoryData{}, false
}
size := ll.Len()
if size == 0 {
return []*dataobj.RRDData{}, false
return []*dataobj.HistoryData{}, false
}
firstElement := ll.Front()
firstItem := firstElement.Value.(*dataobj.JudgeItem)
var vs []*dataobj.RRDData
var vs []*dataobj.HistoryData
isEnough := true
judgeType := firstItem.DsType[0]
@ -79,15 +79,21 @@ func (ll *SafeLinkedList) HistoryData(limit int) ([]*dataobj.RRDData, bool) {
limit = size
isEnough = false
}
vs = make([]*dataobj.RRDData, limit)
vs[0] = &dataobj.RRDData{Timestamp: firstItem.Timestamp, Value: dataobj.JsonFloat(firstItem.Value)}
vs = make([]*dataobj.HistoryData, limit)
vs[0] = &dataobj.HistoryData{
Timestamp: firstItem.Timestamp,
Value: dataobj.JsonFloat(firstItem.Value),
Extra: firstItem.Extra,
}
i := 1
currentElement := firstElement
for i < limit {
nextElement := currentElement.Next()
vs[i] = &dataobj.RRDData{
vs[i] = &dataobj.HistoryData{
Timestamp: nextElement.Value.(*dataobj.JudgeItem).Timestamp,
Value: dataobj.JsonFloat(nextElement.Value.(*dataobj.JudgeItem).Value),
Extra: nextElement.Value.(*dataobj.JudgeItem).Extra,
}
i++
currentElement = nextElement

28
src/modules/judge/judge/func.go
View File

@ -8,7 +8,7 @@ import (
)
type Function interface {
Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool)
Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool)
}
type MaxFunction struct {
@ -18,7 +18,7 @@ type MaxFunction struct {
RightValue float64
}
func (this MaxFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this MaxFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -42,7 +42,7 @@ type MinFunction struct {
RightValue float64
}
func (this MinFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this MinFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -66,7 +66,7 @@ type AllFunction struct {
RightValue float64
}
func (this AllFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this AllFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -90,7 +90,7 @@ type SumFunction struct {
RightValue float64
}
func (this SumFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this SumFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -112,7 +112,7 @@ type AvgFunction struct {
RightValue float64
}
func (this AvgFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this AvgFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -135,7 +135,7 @@ type DiffFunction struct {
}
// 只要有一个点的diff触发阈值，就报警
func (this DiffFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this DiffFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -163,7 +163,7 @@ type PDiffFunction struct {
RightValue float64
}
func (this PDiffFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this PDiffFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -193,7 +193,7 @@ type HappenFunction struct {
RightValue float64
}
func (this HappenFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this HappenFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
for n, i := 0, 0; i < len(vs); i++ {
if checkIsTriggered(vs[i].Value, this.Operator, this.RightValue) {
n++
@ -211,7 +211,7 @@ type NodataFunction struct {
Function
}
func (this NodataFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this NodataFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
for _, value := range vs {
if !math.IsNaN(float64(value.Value)) {
return value.Value, false
@ -228,7 +228,7 @@ type CAvgAbsFunction struct {
CompareValue float64
}
func (this CAvgAbsFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this CAvgAbsFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -253,7 +253,7 @@ type CAvgFunction struct {
CompareValue float64
}
func (this CAvgFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this CAvgFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -277,7 +277,7 @@ type CAvgRateAbsFunction struct {
CompareValue float64
}
func (this CAvgRateAbsFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this CAvgRateAbsFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}
@ -302,7 +302,7 @@ type CAvgRateFunction struct {
CompareValue float64
}
func (this CAvgRateFunction) Compute(vs []*dataobj.RRDData) (leftValue dataobj.JsonFloat, isTriggered bool) {
func (this CAvgRateFunction) Compute(vs []*dataobj.HistoryData) (leftValue dataobj.JsonFloat, isTriggered bool) {
if len(vs) < this.Limit {
return
}

22
src/modules/judge/judge/judge.go
View File

@ -62,7 +62,7 @@ func ToJudge(historyMap *cache.JudgeItemMap, key string, val *dataobj.JudgeItem,
Judge(stra, stra.Exprs, historyData, val, now, history, "", "", "", []bool{})
}
func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.HistoryData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
stats.Counter.Set("running", 1)
if len(exps) < 1 {
@ -86,10 +86,6 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
Granularity: int(firstItem.Step),
Points: historyData,
}
if len(history) == 0 {
//只有第一个指标是push的模式，可以获取到extra字段
h.Extra = firstItem.Extra
}
history = append(history, h)
defer func() {
@ -135,7 +131,7 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
Tags: "",
DsType: "GAUGE",
}
Judge(stra, exps[1:], []*dataobj.RRDData{}, judgeItem, now, history, info, value, extra, status)
Judge(stra, exps[1:], []*dataobj.HistoryData{}, judgeItem, now, history, info, value, extra, status)
return
}
@ -143,7 +139,7 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
firstItem.Endpoint = respData[i].Endpoint
firstItem.Tags = getTags(respData[i].Counter)
firstItem.Step = respData[i].Step
Judge(stra, exps[1:], respData[i].Values, firstItem, now, history, info, value, extra, status)
Judge(stra, exps[1:], dataobj.RRDData2HistoryData(respData[i].Values), firstItem, now, history, info, value, extra, status)
}
} else {
@ -162,13 +158,13 @@ func Judge(stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, f
firstItem.Endpoint = respData[i].Endpoint
firstItem.Tags = getTags(respData[i].Counter)
firstItem.Step = respData[i].Step
Judge(stra, exps[1:], respData[i].Values, firstItem, now, history, info, value, extra, status)
Judge(stra, exps[1:], dataobj.RRDData2HistoryData(respData[i].Values), firstItem, now, history, info, value, extra, status)
}
}
}
}
func judgeItemWithStrategy(stra *model.Stra, historyData []*dataobj.RRDData, exp model.Exp, firstItem *dataobj.JudgeItem, now int64) (leftValue dataobj.JsonFloat, isTriggered bool) {
func judgeItemWithStrategy(stra *model.Stra, historyData []*dataobj.HistoryData, exp model.Exp, firstItem *dataobj.JudgeItem, now int64) (leftValue dataobj.JsonFloat, isTriggered bool) {
straFunc := exp.Func
straParam := []interface{}{}
@ -298,7 +294,7 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([]
stats.Counter.Set("get.index", 1)
indexsData, err := query.Xclude(req)
if err != nil {
return reqs, err
logger.Warning("get index err:", err)
}
lostSeries := []cache.Series{}
@ -345,6 +341,10 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([]
}
seriess := cache.SeriesMap.Get(stra.Id)
if len(seriess) == 0 && err != nil {
return reqs, err
}
step := 0
if len(seriess) > 1 {
step = seriess[0].Step
@ -389,7 +389,7 @@ func GetReqs(stra *model.Stra, metric string, endpoints []string, now int64) ([]
return reqs, nil
}
func sendEventIfNeed(historyData []*dataobj.RRDData, status []bool, event *dataobj.Event, stra *model.Stra) {
func sendEventIfNeed(historyData []*dataobj.HistoryData, status []bool, event *dataobj.Event, stra *model.Stra) {
isTriggered := true
for _, s := range status {
isTriggered = isTriggered && s

6
src/modules/judge/judge/nodata.go
View File

@ -51,7 +51,7 @@ func nodataJudge() {
}
nodataJob.Acquire()
go AsyncJudge(nodataJob, stra, stra.Exprs, []*dataobj.RRDData{}, judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
go AsyncJudge(nodataJob, stra, stra.Exprs, []*dataobj.HistoryData{}, judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
}
return
}
@ -80,12 +80,12 @@ func nodataJudge() {
}
nodataJob.Acquire()
go AsyncJudge(nodataJob, stra, stra.Exprs, data.Values, judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
go AsyncJudge(nodataJob, stra, stra.Exprs, dataobj.RRDData2HistoryData(data.Values), judgeItem, now, []dataobj.History{}, "", "", "", []bool{})
}
}
}
func AsyncJudge(sema *semaphore.Semaphore, stra *model.Stra, exps []model.Exp, historyData []*dataobj.RRDData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
func AsyncJudge(sema *semaphore.Semaphore, stra *model.Stra, exps []model.Exp, historyData []*dataobj.HistoryData, firstItem *dataobj.JudgeItem, now int64, history []dataobj.History, info string, value string, extra string, status []bool) {
defer sema.Release()
Judge(stra, exps, historyData, firstItem, now, history, info, value, extra, status)
}

10
src/modules/monapi/cron/event_consumer.go
View File

@ -220,10 +220,9 @@ func isInConverge(event *model.Event) bool {
return false
}
// 三种情况，不需要升级报警
// 两种情况，不需要升级报警
// 1，认领的报警不需要升级
// 2，忽略的报警不需要升级
// 3，屏蔽的报警不需要升级，屏蔽判断在前面已经有了处理，这个方法不用关注
// 2，屏蔽的报警不需要升级，屏蔽判断在前面已经有了处理，这个方法不用关注
func needUpgrade(event *model.Event) bool {
alertUpgradeKey := PrefixAlertUpgrade + fmt.Sprint(event.HashId)
eventAlertKey := PrefixAlertTime + fmt.Sprint(event.HashId)
@ -282,11 +281,6 @@ func needUpgrade(event *model.Event) bool {
return false
}
// 告警已经忽略了
if eventCur.IgnoreAlert == 1 {
return false
}
// 告警之后，比如30分钟没有处理，就需要升级，那首先得知道首次告警时间
if !redisc.HasKey(eventAlertKey) {
err := redisc.SetWithTTL(eventAlertKey, now, 30*24*3600)

17
src/modules/monapi/http/routes/event.go
View File

@ -9,6 +9,7 @@ import (
"github.com/toolkits/pkg/errors"
"github.com/didi/nightingale/src/dataobj"
"github.com/didi/nightingale/src/model"
jsoniter "github.com/json-iterator/go"
)
@ -79,9 +80,9 @@ func eventCurGets(c *gin.Context) {
err = json.Unmarshal([]byte(events[i].Detail), &detail)
errors.Dangerous(err)
tagsList := []string{}
for k, v := range detail[0].Tags {
tagsList = append(tagsList, fmt.Sprintf("%s=%s", k, v))
var tags string
if len(detail) > 0 {
tags = dataobj.SortedTags(detail[0].Tags)
}
alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade)
@ -106,7 +107,7 @@ func eventCurGets(c *gin.Context) {
Etime: events[i].Etime,
Value: events[i].Value,
Info: events[i].Info,
Tags: strings.Join(tagsList, ","),
Tags: tags,
Created: events[i].Created,
Nid: events[i].Nid,
Users: users,
@ -164,9 +165,9 @@ func eventHisGets(c *gin.Context) {
err = json.Unmarshal([]byte(events[i].Detail), &detail)
errors.Dangerous(err)
tagsList := []string{}
for k, v := range detail[0].Tags {
tagsList = append(tagsList, fmt.Sprintf("%s=%s", k, v))
var tags string
if len(detail) > 0 {
tags = dataobj.SortedTags(detail[0].Tags)
}
alertUpgrade, err := model.EventAlertUpgradeUnMarshal(events[i].AlertUpgrade)
@ -191,7 +192,7 @@ func eventHisGets(c *gin.Context) {
Etime: events[i].Etime,
Value: events[i].Value,
Info: events[i].Info,
Tags: strings.Join(tagsList, ","),
Tags: tags,
Created: events[i].Created,
Nid: events[i].Nid,
Users: users,