From 7892a8d2681ef49e3819257ac8d60bca50bd8083 Mon Sep 17 00:00:00 2001 From: kongfei Date: Mon, 15 Aug 2022 18:01:26 +0800 Subject: [PATCH] add ipvs plugin --- conf/input.ipvs/ipvs.toml | 3 + go.mod | 7 +- go.sum | 9 ++ inputs/ipvs/README.md | 85 +++++++++++++++++++ inputs/ipvs/ipvs.go | 165 ++++++++++++++++++++++++++++++++++++ inputs/ipvs/ipvs_nolinux.go | 4 + 6 files changed, 271 insertions(+), 2 deletions(-) create mode 100644 conf/input.ipvs/ipvs.toml create mode 100644 inputs/ipvs/README.md create mode 100644 inputs/ipvs/ipvs.go create mode 100644 inputs/ipvs/ipvs_nolinux.go diff --git a/conf/input.ipvs/ipvs.toml b/conf/input.ipvs/ipvs.toml new file mode 100644 index 0000000..627dd38 --- /dev/null +++ b/conf/input.ipvs/ipvs.toml @@ -0,0 +1,3 @@ +# Collect virtual and real server stats from Linux IPVS +[[instances]] +# no configuration \ No newline at end of file diff --git a/go.mod b/go.mod index f0e48c2..7183103 100644 --- a/go.mod +++ b/go.mod @@ -24,11 +24,14 @@ require ( github.com/hashicorp/consul/api v1.13.0 github.com/influxdata/line-protocol/v2 v2.2.1 github.com/jmoiron/sqlx v1.3.5 + github.com/json-iterator/go v1.1.12 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7 github.com/krallistic/kazoo-go v0.0.0-20170526135507-a15279744f4e + github.com/mattn/go-isatty v0.0.14 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 github.com/miekg/dns v1.1.50 + github.com/moby/ipvs v1.0.2 github.com/oklog/run v1.1.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/alibabacloudlogserviceexporter v0.54.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/jaegerexporter v0.54.0 @@ -187,7 +190,6 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/klauspost/compress v1.15.6 // indirect github.com/knadh/koanf v1.4.2 // indirect @@ -198,7 +200,6 @@ require ( github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -251,6 +252,8 @@ require ( github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/ugorji/go/codec v1.1.7 // indirect + github.com/vishvananda/netlink v1.1.0 // indirect + github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect github.com/vultr/govultr/v2 v2.17.2 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.1 // indirect diff --git a/go.sum b/go.sum index 035d175..bdf802a 100644 --- a/go.sum +++ b/go.sum @@ -847,6 +847,8 @@ github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= +github.com/moby/ipvs v1.0.2 h1:NSbzuRTvfneftLU3VwPU5QuA6NZ0IUmqq9+VHcQxqHw= +github.com/moby/ipvs v1.0.2/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1158,6 +1160,10 @@ github.com/ulricqin/gosnmp v0.0.1/go.mod h1:9OasJbP94MjBGOLNghlVwgG3UN05ATurou1G github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -1444,6 +1450,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1568,6 +1575,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1862,6 +1870,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I= gotest.tools/v3 v3.2.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/inputs/ipvs/README.md b/inputs/ipvs/README.md new file mode 100644 index 0000000..e9d58fb --- /dev/null +++ b/inputs/ipvs/README.md @@ -0,0 +1,85 @@ +# ipvs + +Forked from telegraf. The IPVS input plugin uses the linux kernel netlink socket interface to gather +metrics about ipvs virtual and real servers. +**Supported Platforms:** Linux + +### Permissions + +Assuming you installed the telegraf package via one of the published packages, +the process will be running as the `telegraf` user. However, in order for this +plugin to communicate over netlink sockets it needs the telegraf process to be +running as `root` (or some user with `CAP_NET_ADMIN` and `CAP_NET_RAW`). Be sure +to ensure these permissions before running telegraf with this plugin included. + +## Configuration +``` +# Collect virtual and real server stats from Linux IPVS +[[instances]] + # no configuration +``` + +## Metrics + +Server will contain tags identifying how it was configured, using one of +`address` + `port` + `protocol` *OR* `fwmark`. This is how one would normally +configure a virtual server using `ipvsadm`. + +- ipvs_virtual_server + - tags: + - sched (the scheduler in use) + - netmask (the mask used for determining affinity) + - address_family (inet/inet6) + - address + - port + - protocol + - fwmark + - fields: + - connections + - pkts_in + - pkts_out + - bytes_in + - bytes_out + - pps_in + - pps_out + - cps + +- ipvs_real_server + - tags: + - address + - port + - address_family (inet/inet6) + - virtual_address + - virtual_port + - virtual_protocol + - virtual_fwmark + - fields: + - active_connections + - inactive_connections + - connections + - pkts_in + - pkts_out + - bytes_in + - bytes_out + - pps_in + - pps_out + - cps + +## Example Output + +Virtual server is configured using `fwmark` and backed by 2 real servers: + +```shell +ipvs_virtual_server,address=172.18.64.234,address_family=inet,netmask=32,port=9000,protocol=tcp,sched=rr bytes_in=0i,bytes_out=0i,pps_in=0i,pps_out=0i,cps=0i,connections=0i,pkts_in=0i,pkts_out=0i 1541019340000000000 +ipvs_real_server,address=172.18.64.220,address_family=inet,port=9000,virtual_address=172.18.64.234,virtual_port=9000,virtual_protocol=tcp active_connections=0i,inactive_connections=0i,pkts_in=0i,bytes_out=0i,pps_out=0i,connections=0i,pkts_out=0i,bytes_in=0i,pps_in=0i,cps=0i 1541019340000000000 +ipvs_real_server,address=172.18.64.219,address_family=inet,port=9000,virtual_address=172.18.64.234,virtual_port=9000,virtual_protocol=tcp active_connections=0i,inactive_connections=0i,pps_in=0i,pps_out=0i,connections=0i,pkts_in=0i,pkts_out=0i,bytes_in=0i,bytes_out=0i,cps=0i 1541019340000000000 +``` + +Virtual server is configured using `proto+addr+port` and backed by 2 real +servers: + +```shell +ipvs_virtual_server,address_family=inet,fwmark=47,netmask=32,sched=rr cps=0i,connections=0i,pkts_in=0i,pkts_out=0i,bytes_in=0i,bytes_out=0i,pps_in=0i,pps_out=0i 1541019340000000000 +ipvs_real_server,address=172.18.64.220,address_family=inet,port=9000,virtual_fwmark=47 inactive_connections=0i,pkts_out=0i,bytes_out=0i,pps_in=0i,cps=0i,active_connections=0i,pkts_in=0i,bytes_in=0i,pps_out=0i,connections=0i 1541019340000000000 +ipvs_real_server,address=172.18.64.219,address_family=inet,port=9000,virtual_fwmark=47 cps=0i,active_connections=0i,inactive_connections=0i,connections=0i,pkts_in=0i,bytes_out=0i,pkts_out=0i,bytes_in=0i,pps_in=0i,pps_out=0i 1541019340000000000 +``` \ No newline at end of file diff --git a/inputs/ipvs/ipvs.go b/inputs/ipvs/ipvs.go new file mode 100644 index 0000000..cc744ec --- /dev/null +++ b/inputs/ipvs/ipvs.go @@ -0,0 +1,165 @@ +//go:build linux +// +build linux + +package ipvs + +import ( + _ "embed" + "fmt" + "log" + "math/bits" + "strconv" + "syscall" + + "github.com/moby/ipvs" + + "flashcat.cloud/categraf/config" + "flashcat.cloud/categraf/inputs" + "flashcat.cloud/categraf/types" +) + +const inputName = "ipvs" + +type IPVS struct { + config.PluginConfig + Instances []*Instance `toml:"instances"` +} + +func init() { + inputs.Add(inputName, func() inputs.Input { + return &IPVS{} + }) +} + +func (l *IPVS) GetInstances() []inputs.Instance { + ret := make([]inputs.Instance, len(l.Instances)) + for i := 0; i < len(l.Instances); i++ { + ret[i] = l.Instances[i] + } + return ret +} + +// IPVS holds the state for this input plugin +type Instance struct { + config.InstanceConfig + + handle *ipvs.Handle +} + +// Gather gathers the stats +func (i *Instance) Gather(slist *types.SampleList) error { + if i.handle == nil { + h, err := ipvs.New("") + if err != nil { + return fmt.Errorf("unable to open IPVS handle: %v", err) + } + i.handle = h + } + + services, err := i.handle.GetServices() + if err != nil { + i.handle.Close() + i.handle = nil // trigger a reopen on next call to gather + return fmt.Errorf("failed to list IPVS services: %v", err) + } + for _, s := range services { + fields := map[string]interface{}{ + "connections": s.Stats.Connections, + "pkts_in": s.Stats.PacketsIn, + "pkts_out": s.Stats.PacketsOut, + "bytes_in": s.Stats.BytesIn, + "bytes_out": s.Stats.BytesOut, + "pps_in": s.Stats.PPSIn, + "pps_out": s.Stats.PPSOut, + "cps": s.Stats.CPS, + } + slist.PushSamples(inputName, fields, serviceTags(s)) + + destinations, err := i.handle.GetDestinations(s) + if err != nil { + log.Printf("E! Failed to list destinations for a virtual server: %v\n", err) + continue // move on to the next virtual server + } + + for _, d := range destinations { + fields := map[string]interface{}{ + "active_connections": d.ActiveConnections, + "inactive_connections": d.InactiveConnections, + "connections": d.Stats.Connections, + "pkts_in": d.Stats.PacketsIn, + "pkts_out": d.Stats.PacketsOut, + "bytes_in": d.Stats.BytesIn, + "bytes_out": d.Stats.BytesOut, + "pps_in": d.Stats.PPSIn, + "pps_out": d.Stats.PPSOut, + "cps": d.Stats.CPS, + } + destTags := destinationTags(d) + if s.FWMark > 0 { + destTags["virtual_fwmark"] = strconv.Itoa(int(s.FWMark)) + } else { + destTags["virtual_protocol"] = protocolToString(s.Protocol) + destTags["virtual_address"] = s.Address.String() + destTags["virtual_port"] = strconv.Itoa(int(s.Port)) + } + // acc.AddGauge("ipvs_real_server", fields, destTags) + slist.PushSamples(inputName, fields, destTags) + } + } + + return nil +} + +// helper: given a Service, return tags that identify it +func serviceTags(s *ipvs.Service) map[string]string { + ret := map[string]string{ + "sched": s.SchedName, + "netmask": strconv.Itoa(bits.OnesCount32(s.Netmask)), + "address_family": addressFamilyToString(s.AddressFamily), + } + // Per the ipvsadm man page, a virtual service is defined "based on + // protocol/addr/port or firewall mark" + if s.FWMark > 0 { + ret["fwmark"] = strconv.Itoa(int(s.FWMark)) + } else { + ret["protocol"] = protocolToString(s.Protocol) + ret["address"] = s.Address.String() + ret["port"] = strconv.Itoa(int(s.Port)) + } + return ret +} + +// helper: given a Destination, return tags that identify it +func destinationTags(d *ipvs.Destination) map[string]string { + return map[string]string{ + "address": d.Address.String(), + "port": strconv.Itoa(int(d.Port)), + "address_family": addressFamilyToString(d.AddressFamily), + } +} + +// helper: convert protocol uint16 to human readable string (if possible) +func protocolToString(p uint16) string { + switch p { + case syscall.IPPROTO_TCP: + return "tcp" + case syscall.IPPROTO_UDP: + return "udp" + case syscall.IPPROTO_SCTP: + return "sctp" + default: + return fmt.Sprintf("%d", p) + } +} + +// helper: convert addressFamily to a human readable string +func addressFamilyToString(af uint16) string { + switch af { + case syscall.AF_INET: + return "inet" + case syscall.AF_INET6: + return "inet6" + default: + return fmt.Sprintf("%d", af) + } +} diff --git a/inputs/ipvs/ipvs_nolinux.go b/inputs/ipvs/ipvs_nolinux.go new file mode 100644 index 0000000..b46035f --- /dev/null +++ b/inputs/ipvs/ipvs_nolinux.go @@ -0,0 +1,4 @@ +//go:build !linux +// +build !linux + +package ipvs