From e0cc7dbffa40323eefe9adae16cf4081152038ed Mon Sep 17 00:00:00 2001 From: 710leo <710leo@gmail.com> Date: Tue, 29 Jun 2021 20:09:40 +0800 Subject: [PATCH] refactor: log regex check api --- http/router.go | 2 +- http/router_collect_rule.go | 156 +++++++++--------------------------- 2 files changed, 38 insertions(+), 120 deletions(-) diff --git a/http/router.go b/http/router.go index f493cff5..c3e3945a 100644 --- a/http/router.go +++ b/http/router.go @@ -146,7 +146,7 @@ func configRoutes(r *gin.Engine) { pages.POST("/collect-rules", login(), collectRuleAdd) pages.DELETE("/collect-rules", login(), collectRuleDel) pages.PUT("/collect-rule/:id", login(), collectRulePut) - pages.POST("/log/check", regExpCheck) // check collect rule + pages.POST("/log/check", regExpCheck) pages.GET("/metric-descriptions", metricDescriptionGets) pages.POST("/metric-descriptions", login(), metricDescriptionAdd) diff --git a/http/router_collect_rule.go b/http/router_collect_rule.go index e7f681cc..e3473bf8 100644 --- a/http/router_collect_rule.go +++ b/http/router_collect_rule.go @@ -1,16 +1,15 @@ package http import ( - "fmt" "regexp" "strings" "time" "github.com/gin-gonic/gin" - "github.com/toolkits/pkg/logger" "github.com/didi/nightingale/v5/cache" "github.com/didi/nightingale/v5/models" + "github.com/didi/nightingale/v5/pkg/i18n" ) type collectRuleForm struct { @@ -136,8 +135,6 @@ type RegExpCheck struct { Data []map[string]string `json:"tags"` } -var RegExpExcludePatition string = "```EXCLUDE```" - func regExpCheck(c *gin.Context) { param := make(map[string]string) dangerous(c.ShouldBind(¶m)) @@ -147,75 +144,45 @@ func regExpCheck(c *gin.Context) { Data: make([]map[string]string, 0), } - // 处理时间格式 - if t, ok := param["time"]; !ok || t == "" { - tmp := map[string]string{"time": "time参数不存在或为空"} + calcMethod := param["func"] + if calcMethod == "" { + tmp := map[string]string{"func": "is empty"} ret.Data = append(ret.Data, tmp) - } else { - timePat, _ := GetPatAndTimeFormat(param["time"]) - if timePat == "" { - tmp := map[string]string{"time": genErrMsg("时间格式")} - ret.Data = append(ret.Data, tmp) - } else { - suc, tRes, _ := checkRegPat(timePat, param["log"], true) - if !suc { - ret.Success = false - tRes = genErrMsg("时间格式") - } - tmp := map[string]string{"time": tRes} - ret.Data = append(ret.Data, tmp) - } + renderData(c, ret, nil) + return } - // 计算方式 - calc_method := param["calc_method"] - - // 处理主正则(with exclude) + // 处理主正则 if re, ok := param["re"]; !ok || re == "" { - tmp := map[string]string{"re": "re参数不存在或为空"} + tmp := map[string]string{"re": "regex does not exist or is empty"} ret.Data = append(ret.Data, tmp) - } else { - // 处理exclude的情况 - exclude := "" - if strings.Contains(re, RegExpExcludePatition) { - l := strings.Split(re, RegExpExcludePatition) - if len(l) >= 2 { - param["re"] = l[0] - exclude = l[1] - } - } - - // 匹配主正则 - suc, reRes, isSub := checkRegPat(param["re"], param["log"], false) - if !suc { - ret.Success = false - reRes = genErrMsg("主正则") - } - if calc_method != "" && calc_method != "cnt" && !isSub { - ret.Success = false - reRes = genSubErrMsg("主正则") - } - tmp := map[string]string{"主正则": reRes} - ret.Data = append(ret.Data, tmp) - - // 匹配exclude, 这个不影响失败 - if exclude != "" { - suc, exRes, _ := checkRegPat(exclude, param["log"], false) - if !suc { - //ret.Success = false - exRes = "未匹配到排除串,请检查是否符合预期" - } - tmp := map[string]string{"排除串": exRes} - ret.Data = append(ret.Data, tmp) - } + renderData(c, ret, nil) + return } + // 匹配主正则 + suc, reRes, isSub := checkRegex(param["re"], param["log"]) + if !suc { + ret.Success = false + reRes = genErrMsg("re") + ret.Data = append(ret.Data, map[string]string{"re": reRes}) + renderData(c, ret, nil) + return + } + if calcMethod == "histogram" && !isSub { + ret.Success = false + reRes = genSubErrMsg("re") + ret.Data = append(ret.Data, map[string]string{"re": reRes}) + renderData(c, ret, nil) + return + } + + ret.Data = append(ret.Data, map[string]string{"re": reRes}) // 处理tags var nonTagKey = map[string]bool{ - "re": true, - "log": true, - "time": true, - "calc_method": true, + "re": true, + "log": true, + "func": true, } for tagk, pat := range param { @@ -223,7 +190,7 @@ func regExpCheck(c *gin.Context) { if _, ok := nonTagKey[tagk]; ok { continue } - suc, tagRes, isSub := checkRegPat(pat, param["log"], false) + suc, tagRes, isSub := checkRegex(pat, param["log"]) if !suc { // 正则错误 ret.Success = false @@ -245,49 +212,8 @@ func regExpCheck(c *gin.Context) { renderData(c, ret, nil) } -//根据配置的时间格式，获取对应的正则匹配pattern和time包用的时间格式 -func GetPatAndTimeFormat(tf string) (string, string) { - var pat, timeFormat string - switch tf { - case "dd/mmm/yyyy:HH:MM:SS": - pat = `([012][0-9]|3[01])/[JFMASOND][a-z]{2}/(2[0-9]{3}):([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "02/Jan/2006:15:04:05" - case "dd/mmm/yyyy HH:MM:SS": - pat = `([012][0-9]|3[01])/[JFMASOND][a-z]{2}/(2[0-9]{3})\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "02/Jan/2006 15:04:05" - case "yyyy-mm-ddTHH:MM:SS": - pat = `(2[0-9]{3})-(0[1-9]|1[012])-([012][0-9]|3[01])T([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "2006-01-02T15:04:05" - case "dd-mmm-yyyy HH:MM:SS": - pat = `([012][0-9]|3[01])-[JFMASOND][a-z]{2}-(2[0-9]{3})\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "02-Jan-2006 15:04:05" - case "yyyy-mm-dd HH:MM:SS": - pat = `(2[0-9]{3})-(0[1-9]|1[012])-([012][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "2006-01-02 15:04:05" - case "yyyy/mm/dd HH:MM:SS": - pat = `(2[0-9]{3})/(0[1-9]|1[012])/([012][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "2006/01/02 15:04:05" - case "yyyymmdd HH:MM:SS": - pat = `(2[0-9]{3})(0[1-9]|1[012])([012][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "20060102 15:04:05" - case "mmm dd HH:MM:SS": - pat = `[JFMASOND][a-z]{2}\s+([1-9]|[1-2][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "Jan 2 15:04:05" - case "mmdd HH:MM:SS": - pat = `(0[1-9]|1[012])([012][0-9]|3[01])\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "0102 15:04:05" - case "dd mmm yyyy HH:MM:SS": - pat = `([012][0-9]|3[01])\s+[JFMASOND][a-z]{2}\s+(2[0-9]{3})\s([01][0-9]|2[0-4])(:[012345][0-9]){2}` - timeFormat = "02 Jan 2006 15:04:05" - default: - logger.Errorf("match time pac failed : [timeFormat:%s]", tf) - return "", "" - } - return pat, timeFormat -} - // 出错信息直接放在body里 -func checkRegPat(pat string, log string, origin bool) (succ bool, result string, isSub bool) { +func checkRegex(pat string, log string) (succ bool, result string, isSub bool) { if pat == "" { return false, "", false } @@ -307,15 +233,7 @@ func checkRegPat(pat string, log string, origin bool) (succ bool, result string, return true, res[0], false // 查到了，默认取第一个串 default: - var msg string - if origin { - msg = res[0] - isSub = false - } else { - msg = res[1] - isSub = true - } - return true, msg, isSub + return true, res[1], true } } @@ -326,15 +244,15 @@ func includeIllegalChar(s string) bool { // 生成返回错误信息 func genErrMsg(sign string) string { - return fmt.Sprintf("正则匹配失败，请认真检查您[%s]的配置", sign) + return i18n.Sprintf("regular match failed, please check your configuration:[%s]", sign) } // 生成子串匹配错误信息 func genSubErrMsg(sign string) string { - return fmt.Sprintf("正则匹配成功。但根据配置，并没有获取到()内的子串，请认真检查您[%s]的配置", sign) + return i18n.Sprintf("regular match was successful. according to the configuration, it does not get the substring inside(), please check your configuration:[%s]", sign) } // 生成子串匹配错误信息 func genIllegalCharErrMsg() string { - return `正则匹配成功。但是tag的key或者value包含非法字符:[:,/=\r\n\t], 请重新调整` + return i18n.Sprintf(`key or value of tag contains illegal characters:[:,/=\r\n\t]`) }