agent 增加默认tags功能, agent 增加正则匹配磁盘挂载类型过滤功能 (#683)

* agent 增加默认tags功能, agent 增加正则匹配磁盘挂载类型过滤功能

* agent 增加默认tags功能, agent 增加正则匹配磁盘挂载类型过滤功能

Co-authored-by: huboc <huboc@zbj.com>

etc/agentd.yml

@@ -9,6 +9,12 @@ enable:
   report: true
   metrics: true
 
+defaultTags:
+  enable: false
+  tags:
+    key1: value1
+    key2: value2
+    
 udp:
   enable: false
   listen: :788
@@ -56,12 +62,16 @@ sys:
     - em
     - ens
 
-  # ignore disk mount point
-  mountIgnore:
-    prefix:
+  # 磁盘采集
+  mountCollect:
+    # 正则匹配需要采集的挂载卷类型
+    typePrefix: "^(btrfs|ext2|ext3|ext4|jfs|reiser|xfs|ffs|ufs|jfs|jfs2|vxfs|hfs|ntfs|fat32|zfs|nfs)$"
+    # 忽略采集以下面开头的挂载目录
+    ignorePrefix:
       - /var/lib
       - /run
-    # collect anyway
+      - /boot
+    # 仍然采集ignorePrefix中的例外的挂载目录(填写全路径)
     exclude: []
 
   ignoreMetrics:

src/modules/agentd/config/config.go

@@ -16,6 +16,7 @@ import (
 
 type ConfigT struct {
 	Logger  loggeri.Config `yaml:"logger"`
+	DefaultTags   tagsSection    `yaml:"defaultTags"`
 	Stra    straSection    `yaml:"stra"`
 	Worker  workerSection  `yaml:"worker"`
 	Sys     sys.SysSection `yaml:"sys"`
@@ -26,6 +27,11 @@ type ConfigT struct {
 	Metrics MetricsSection `yaml:"metrics"`
 }
 
+type tagsSection struct {
+	Enable   bool                `yaml:"enable"`
+	Tags     map[string]string   `yaml:"tags"`
+}
+
 type UdpSection struct {
 	Enable bool   `yaml:"enable"`
 	Listen string `yaml:"listen"`

src/modules/agentd/core/push.go

@@ -2,12 +2,14 @@ package core
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"math/rand"
 	"net"
 	"net/rpc"
 	"reflect"
+	"strings"
 	"time"
 
 	"github.com/didi/nightingale/v4/src/common/address"
@@ -23,6 +25,30 @@ func Push(metricItems []*dataobj.MetricValue) error {
 	var err error
 	var items []*dataobj.MetricValue
 	now := time.Now().Unix()
+	dt := config.Config.DefaultTags.Tags
+	if config.Config.DefaultTags.Enable && len(dt) > 0 {
+		var buf bytes.Buffer
+		defaultTagsList := []string{}
+		for k, v := range dt {
+			buf.Reset()
+			buf.WriteString(k)
+			buf.WriteString("=")
+			buf.WriteString(v)
+			defaultTagsList = append(defaultTagsList, buf.String())
+		}
+		defaultTags := strings.Join(defaultTagsList, ",")
+		for i, x := range metricItems {
+			buf.Reset()
+			if x.Tags == "" {
+				metricItems[i].Tags = defaultTags
+			} else {
+				buf.WriteString(metricItems[i].Tags)
+				buf.WriteString(",")
+				buf.WriteString(defaultTags)
+				metricItems[i].Tags = buf.String()
+			}
+		}
+	}
 
 	for _, item := range metricItems {
 		logger.Debugf("->recv:%+v", item)

src/modules/agentd/sys/config.go

@@ -3,7 +3,7 @@ package sys
 type SysSection struct {
 	Enable           bool                `yaml:"enable"`
 	IfacePrefix      []string            `yaml:"ifacePrefix"`
-	MountIgnore      MountIgnoreSection  `yaml:"mountIgnore"`
+	MountCollect     MountSection        `yaml:"mountCollect"`
 	IgnoreMetrics    []string            `yaml:"ignoreMetrics"`
 	IgnoreMetricsMap map[string]struct{} `yaml:"-"`
 	NtpServers       []string            `yaml:"ntpServers"`
@@ -14,8 +14,9 @@ type SysSection struct {
 	FsRWEnable       bool                `yaml:"fsRWEnable"`
 }
 
-type MountIgnoreSection struct {
-	Prefix  []string `yaml:"prefix"`
+type MountSection struct {
+	TypePrefix    string    `yaml:"typePrefix"`
+	IgnorePrefix  []string  `yaml:"ignorePrefix"`
 	Exclude []string `yaml:"exclude"`
 }
 

src/modules/agentd/sys/funcs/dfstat.go

@@ -16,6 +16,7 @@ package funcs
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/didi/nightingale/v4/src/common/dataobj"
@@ -42,6 +43,16 @@ func DeviceMetrics() []*dataobj.MetricValue {
 
 	for idx := range mountPoints {
 		fsSpec, fsFile, fsVfstype := mountPoints[idx][0], mountPoints[idx][1], mountPoints[idx][2]
+		// 判断挂载卷类型是否匹配，不匹配则跳过
+		if !IsFsVfstypeValid(sys.Config.MountCollect.TypePrefix, fsVfstype) {
+			continue
+		}
+		
+		// 注意: 虽然前缀被忽略了，但是被忽略的这部分分区里边有些仍然是需要采集的
+		if hasIgnorePrefix(fsFile, sys.Config.MountCollect.IgnorePrefix) &&
+			!slice.ContainsString(sys.Config.MountCollect.Exclude, fsFile) {
+			continue
+		}
 
 		if _, exists := fsFileFilter[fsFile]; exists {
 			logger.Debugf("mount point %s was collected", fsFile)
@@ -50,12 +61,6 @@ func DeviceMetrics() []*dataobj.MetricValue {
 			fsFileFilter[fsFile] = struct{}{}
 		}
 
-		// 注意: 虽然前缀被忽略了，但是被忽略的这部分分区里边有些仍然是需要采集的
-		if hasIgnorePrefix(fsFile, sys.Config.MountIgnore.Prefix) &&
-			!slice.ContainsString(sys.Config.MountIgnore.Exclude, fsFile) {
-			continue
-		}
-
 		var du *nux.DeviceUsage
 		du, err = nux.BuildDeviceUsage(fsSpec, fsFile, fsVfstype)
 		if err != nil {
@@ -96,6 +101,14 @@ func DeviceMetrics() []*dataobj.MetricValue {
 	return ret
 }
 
+func IsFsVfstypeValid(regx string, fsVfstype string) bool {
+	match, err := regexp.MatchString(regx, fsVfstype)
+	if err != nil {
+		return false
+	}
+	return match
+}
+
 func hasIgnorePrefix(fsFile string, ignoreMountPointsPrefix []string) bool {
 	hasPrefix := false
 	if len(ignoreMountPointsPrefix) > 0 {

src/modules/agentd/sys/funcs/fsstat.go

@@ -44,23 +44,28 @@ func FsRWMetrics() []*dataobj.MetricValue {
 	fsFileFilter := make(map[string]struct{}) //过滤 /proc/mounts 出现重复的fsFile
 
 	for idx := range mountPoints {
-		var du *nux.DeviceUsage
-		du, err = nux.BuildDeviceUsage(mountPoints[idx][0], mountPoints[idx][1], mountPoints[idx][2])
-		if err != nil {
-			logger.Warning(idx, " failed to call BuildDeviceUsage:", err)
+		fsSpec, fsFile, fsVfstype := mountPoints[idx][0], mountPoints[idx][1], mountPoints[idx][2]
+		//判断挂载卷类型是否匹配，不匹配则跳过
+		if !IsFsVfstypeValid(sys.Config.MountCollect.TypePrefix, fsVfstype) {
 			continue
 		}
 
-		if hasIgnorePrefix(du.FsFile, sys.Config.MountIgnore.Prefix) &&
-			!slice.ContainsString(sys.Config.MountIgnore.Exclude, du.FsFile) {
+		if hasIgnorePrefix(fsFile, sys.Config.MountCollect.IgnorePrefix) &&
+			!slice.ContainsString(sys.Config.MountCollect.Exclude, fsFile) {
 			continue
 		}
 
-		if _, exists := fsFileFilter[du.FsFile]; exists {
-			logger.Debugf("mount point %s was collected", du.FsFile)
+		if _, exists := fsFileFilter[fsFile]; exists {
+			logger.Debugf("mount point %s was collected", fsFile)
 			continue
 		} else {
-			fsFileFilter[du.FsFile] = struct{}{}
+			fsFileFilter[fsFile] = struct{}{}
+		}
+		var du *nux.DeviceUsage
+		du, err = nux.BuildDeviceUsage(fsSpec, fsFile, fsVfstype)
+		if err != nil {
+			logger.Errorf("fsSpec: %s, fsFile: %s, fsVfstype: %s, failed to call BuildDeviceUsage, error: %v", fsSpec, fsFile, fsVfstype, err)
+			continue
 		}
 
 		tags := fmt.Sprintf("mount=%s", du.FsFile)