From 6b1e432f6d8b70e9b5c962ceb8e6d20b8f774458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=87=95=E5=B0=8F=E4=B9=99?= <907974064@qq.com> Date: Fri, 15 Jan 2021 09:16:06 +0800 Subject: [PATCH 1/4] =?UTF-8?q?m3db=20writetagged=E5=BA=94=E8=AF=A5?= =?UTF-8?q?=E5=B9=B6=E5=8F=91=E5=81=9A=EF=BC=8C=E4=B8=8D=E7=84=B6=E4=BC=9A?= =?UTF-8?q?=E5=AF=BC=E8=87=B4transfer=20rpc=E5=8F=98=E6=85=A2=20(#514)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * m3db writetagged应该并发做，不然会导致transfer rpc变慢 * go func指针传参问题 --- src/modules/transfer/backend/m3db/m3db.go | 39 ++++++++++++++--------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/modules/transfer/backend/m3db/m3db.go b/src/modules/transfer/backend/m3db/m3db.go index 7a66f67a..e4467c74 100644 --- a/src/modules/transfer/backend/m3db/m3db.go +++ b/src/modules/transfer/backend/m3db/m3db.go @@ -3,6 +3,7 @@ package m3db import ( "fmt" "sync" + "sync/atomic" "time" "github.com/didi/nightingale/src/common/dataobj" @@ -89,23 +90,31 @@ func (p *Client) Push2Queue(items []*dataobj.MetricValue) { logger.Errorf("unable to get m3db session: %s", err) return } - - errCnt := 0 + var errCnt int32 + var ( + wg sync.WaitGroup + ) for _, item := range items { - if err := session.WriteTagged( - p.namespaceID, - mvID(item), - ident.NewTagsIterator(mvTags(item)), - time.Unix(item.Timestamp, 0), - item.Value, - xtime.Second, - nil, - ); err != nil { - logger.Errorf("unable to writeTagged: %s", err) - errCnt++ - } + wg.Add(1) + go func(dm *dataobj.MetricValue) { + err := session.WriteTagged( + p.namespaceID, + mvID(dm), + ident.NewTagsIterator(mvTags(dm)), + time.Unix(dm.Timestamp, 0), + dm.Value, + xtime.Second, + nil) + if err != nil { + logger.Errorf("unable to writeTagged: %s", err) + atomic.AddInt32(&errCnt, 1) + } + wg.Done() + }(item) + } - stats.Counter.Set("m3db.queue.err", errCnt) + wg.Wait() + stats.Counter.Set("m3db.queue.err", int(errCnt)) } // QueryData: || (|| endpoints...) (&& tags...) From b00b7817f2ea61d1aadeee4c0dea09fac68f60b8 Mon Sep 17 00:00:00 2001 From: qinyening <710leo@gmail.com> Date: Fri, 15 Jan 2021 15:58:21 +0800 Subject: [PATCH 2/4] Support screen and alert template (#517) --- src/modules/judge/judge.go | 2 +- src/modules/monapi/config/yaml.go | 11 ++++++ src/modules/monapi/http/router.go | 6 ++++ src/modules/monapi/http/router_tpl.go | 49 +++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 src/modules/monapi/http/router_tpl.go diff --git a/src/modules/judge/judge.go b/src/modules/judge/judge.go index 35c80ec1..e7e01100 100644 --- a/src/modules/judge/judge.go +++ b/src/modules/judge/judge.go @@ -117,7 +117,7 @@ func pconf() { func start() { runner.Init() - fmt.Println("transfer start, use configuration file:", *conf) + fmt.Println("judge start, use configuration file:", *conf) fmt.Println("runner.Cwd:", runner.Cwd) fmt.Println("runner.Hostname:", runner.Hostname) } diff --git a/src/modules/monapi/config/yaml.go b/src/modules/monapi/config/yaml.go index 3422ae63..a9d18010 100644 --- a/src/modules/monapi/config/yaml.go +++ b/src/modules/monapi/config/yaml.go @@ -29,6 +29,12 @@ type ConfYaml struct { Link linkSection `yaml:"link"` IndexMod string `yaml:"indexMod"` I18n i18n.I18nSection `yaml:"i18n"` + Tpl tplSection `yaml:"tpl"` +} + +type tplSection struct { + AlertPath string `yaml:"alertPath"` + ScreenPath string `yaml:"screenPath"` } type mergeSection struct { @@ -175,6 +181,11 @@ func Parse(ymlfile string) error { "converge": true, // 历史告警的数据库表，对于已收敛的告警，默认删掉，不保留，省得告警太多 }) + viper.SetDefault("tpl", map[string]string{ + "alertPath": "./etc/alert", + "screenPath": "./etc/screen", + }) + err = viper.Unmarshal(&yaml) if err != nil { return fmt.Errorf("Unmarshal %v", err) diff --git a/src/modules/monapi/http/router.go b/src/modules/monapi/http/router.go index f40ae558..217e0425 100644 --- a/src/modules/monapi/http/router.go +++ b/src/modules/monapi/http/router.go @@ -144,6 +144,12 @@ func Config(r *gin.Engine) { aggr.GET("/:id", aggrCalcGet) } + tpl := r.Group("/api/mon/tpl") + { + tpl.GET("", tplNameGets) + tpl.GET("/content", tplGet) + } + aggrs := r.Group("/api/mon/aggrs").Use() { aggrs.GET("", aggrCalcsWithEndpointGet) diff --git a/src/modules/monapi/http/router_tpl.go b/src/modules/monapi/http/router_tpl.go new file mode 100644 index 00000000..5bcfa06a --- /dev/null +++ b/src/modules/monapi/http/router_tpl.go @@ -0,0 +1,49 @@ +package http + +import ( + "github.com/didi/nightingale/src/modules/monapi/config" + + "github.com/gin-gonic/gin" + "github.com/toolkits/pkg/file" +) + +func tplNameGets(c *gin.Context) { + tplType := mustQueryStr(c, "tplType") + + var files []string + var err error + switch tplType { + case "alert": + files, err = file.FilesUnder(config.Get().Tpl.AlertPath) + dangerous(err) + case "screen": + files, err = file.FilesUnder(config.Get().Tpl.ScreenPath) + dangerous(err) + default: + bomb("tpl type not found") + } + + renderData(c, files, err) +} + +func tplGet(c *gin.Context) { + tplName := mustQueryStr(c, "tplName") + tplType := mustQueryStr(c, "tplType") + + var filePath string + switch tplType { + case "alert": + filePath = config.Get().Tpl.AlertPath + "/" + tplName + case "screen": + filePath = config.Get().Tpl.ScreenPath + "/" + tplName + default: + bomb("tpl type not found") + } + + if !file.IsExist(filePath) { + bomb("tpl not found") + } + + content, err := file.ToString(filePath) + renderData(c, content, err) +} From 51cf58fcdf125ec39a48b89b54930582b64f892d Mon Sep 17 00:00:00 2001 From: alick-liming Date: Fri, 15 Jan 2021 19:54:25 +0800 Subject: [PATCH 3/4] =?UTF-8?q?ams=20agent=E7=9B=B4=E6=8E=A5=E6=8C=82?= =?UTF-8?q?=E8=BD=BD=E5=88=B0=E8=8A=82=E7=82=B9=20(#518)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ams agent直接挂载到节点 * 代码调整 Co-authored-by: alickliming --- src/models/resource.go | 14 +++++- src/modules/ams/http/router.go | 1 + src/modules/ams/http/router_host.go | 72 +++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/src/models/resource.go b/src/models/resource.go index 5f9f2b9f..9103522a 100644 --- a/src/models/resource.go +++ b/src/models/resource.go @@ -292,6 +292,17 @@ func ResourceRegister(hosts []Host, tenant string) error { return err } + // ident agent修改ident带来重复问题 + if res == nil { + ident := hosts[i].Ident + if ident != "" { + res, err = ResourceGet("ident=?", ident) + if err != nil { + return err + } + } + } + if res == nil { res = &Resource{ UUID: uuid, @@ -326,6 +337,7 @@ func ResourceRegister(hosts []Host, tenant string) error { return err } } + res.UUID = uuid res.Ident = hosts[i].Ident res.Name = hosts[i].Name res.Cate = hosts[i].Cate @@ -343,7 +355,7 @@ func ResourceRegister(hosts []Host, tenant string) error { } res.Extend = string(js) - err = res.Update("ident", "name", "cate", "extend", "tenant") + err = res.Update("uuid", "ident", "name", "cate", "extend", "tenant") if err != nil { return err } diff --git a/src/modules/ams/http/router.go b/src/modules/ams/http/router.go index 5932a0e3..d176e6ba 100644 --- a/src/modules/ams/http/router.go +++ b/src/modules/ams/http/router.go @@ -14,6 +14,7 @@ func Config(r *gin.Engine) { userLogin.POST("/hosts", hostPost) userLogin.GET("/host/:id", hostGet) userLogin.PUT("/hosts/tenant", hostTenantPut) + userLogin.PUT("/hosts/node", hostNodePut) userLogin.PUT("/hosts/back", hostBackPut) userLogin.PUT("/hosts/note", hostNotePut) userLogin.PUT("/hosts/cate", hostCatePut) diff --git a/src/modules/ams/http/router_host.go b/src/modules/ams/http/router_host.go index 7bdb5c3a..12cd25d4 100644 --- a/src/modules/ams/http/router_host.go +++ b/src/modules/ams/http/router_host.go @@ -146,6 +146,78 @@ func hostTenantPut(c *gin.Context) { renderMessage(c, err) } +type hostNodeForm struct { + Ids []int64 `json:"ids"` + NodeId int64 `json:"nodeid"` +} + +func (f *hostNodeForm) Validate() { + if len(f.Ids) == 0 { + bomb("ids is empty") + } + + if f.NodeId == 0 { + bomb("nodeid is blank") + } + + if f.NodeId < 0 { + bomb("nodeid is illegal") + } +} + +// 管理员修改主机设备的节点，相当于挂载设备到节点 +func hostNodePut(c *gin.Context) { + var f hostNodeForm + bind(c, &f) + f.Validate() + + loginUser(c).CheckPermGlobal("ams_host_modify") + node, err := models.NodeGet("id=?", f.NodeId) + dangerous(err) + if node == nil { + bomb("node is nil") + } + + if node.Leaf != 1 { + bomb("node is not leaf") + } + + hosts, err := models.HostByIds(f.Ids) + dangerous(err) + if len(hosts) == 0 { + bomb("hosts is empty") + } + + for _, h := range hosts { + if h.Tenant != "" { + bomb("function only for agent first bind, some agent tenant not null, please clear tenant manual") + } + } + + // 绑定租户 + tenant := node.Tenant() + err = models.HostUpdateTenant(f.Ids, tenant) + dangerous(err) + dangerous(models.ResourceRegister(hosts, tenant)) + + // 绑定到节点 + var resUuids []string + for _, id := range f.Ids { + idStr := fmt.Sprintf("host-%d", id) + resUuids = append(resUuids, idStr) + } + if len(resUuids) == 0 { + bomb("res is empty") + } + resIds, err := models.ResourceIdsByUUIDs(resUuids) + dangerous(err) + if len(resIds) == 0 { + bomb("res ids is empty") + } + + renderMessage(c, node.Bind(resIds)) +} + type hostNoteForm struct { Ids []int64 `json:"ids"` Note string `json:"note"` From a4c8638448bcc20c7779d1c7e68e1cd1477dcd22 Mon Sep 17 00:00:00 2001 From: UlricQin Date: Fri, 15 Jan 2021 19:59:26 +0800 Subject: [PATCH 4/4] code refactor --- src/modules/ams/http/router_host.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/ams/http/router_host.go b/src/modules/ams/http/router_host.go index 12cd25d4..cbd61cc2 100644 --- a/src/modules/ams/http/router_host.go +++ b/src/modules/ams/http/router_host.go @@ -190,7 +190,7 @@ func hostNodePut(c *gin.Context) { for _, h := range hosts { if h.Tenant != "" { - bomb("function only for agent first bind, some agent tenant not null, please clear tenant manual") + bomb("%s already belongs to %s", h.Name, h.Tenant) } }