send logs to kafka
This commit is contained in:
kongfei
parent
dbd301c22d
commit
12be2e8a19
|
|
@ -17,7 +17,6 @@ import (
|
|||
logsconfig "flashcat.cloud/categraf/config/logs"
|
||||
"flashcat.cloud/categraf/logs/auditor"
|
||||
"flashcat.cloud/categraf/logs/client"
|
||||
"flashcat.cloud/categraf/logs/client/http"
|
||||
"flashcat.cloud/categraf/logs/diagnostic"
|
||||
"flashcat.cloud/categraf/logs/input/file"
|
||||
"flashcat.cloud/categraf/logs/input/journald"
|
||||
|
|
@ -154,11 +153,7 @@ func (a *Agent) startLogAgent() {
|
|||
return
|
||||
}
|
||||
|
||||
httpConnectivity := logsconfig.HTTPConnectivityFailure
|
||||
if endpoints, err := BuildHTTPEndpoints(intakeTrackType, AgentJSONIntakeProtocol, logsconfig.DefaultIntakeOrigin); err == nil {
|
||||
httpConnectivity = http.CheckConnectivity(endpoints.Main)
|
||||
}
|
||||
endpoints, err := BuildEndpoints(httpConnectivity, intakeTrackType, AgentJSONIntakeProtocol, logsconfig.DefaultIntakeOrigin)
|
||||
endpoints, err := BuildEndpoints(intakeTrackType, AgentJSONIntakeProtocol, logsconfig.DefaultIntakeOrigin)
|
||||
processingRules, err := GlobalProcessingRules()
|
||||
if err != nil {
|
||||
message := fmt.Sprintf("Invalid processing rules: %v", err)
|
||||
|
|
|
|||
|
|
@ -23,20 +23,70 @@ const (
|
|||
)
|
||||
|
||||
// BuildEndpoints returns the endpoints to send logs.
|
||||
func BuildEndpoints(httpConnectivity logsconfig.HTTPConnectivity, intakeTrackType logsconfig.IntakeTrackType, intakeProtocol logsconfig.IntakeProtocol, intakeOrigin logsconfig.IntakeOrigin) (*logsconfig.Endpoints, error) {
|
||||
return BuildEndpointsWithConfig(httpEndpointPrefix, httpConnectivity, intakeTrackType, intakeProtocol, intakeOrigin)
|
||||
func BuildEndpoints(intakeTrackType logsconfig.IntakeTrackType, intakeProtocol logsconfig.IntakeProtocol, intakeOrigin logsconfig.IntakeOrigin) (*logsconfig.Endpoints, error) {
|
||||
return BuildEndpointsWithConfig(httpEndpointPrefix, intakeTrackType, intakeProtocol, intakeOrigin)
|
||||
}
|
||||
|
||||
// BuildEndpointsWithConfig returns the endpoints to send logs.
|
||||
func BuildEndpointsWithConfig(endpointPrefix string, httpConnectivity logsconfig.HTTPConnectivity, intakeTrackType logsconfig.IntakeTrackType, intakeProtocol logsconfig.IntakeProtocol, intakeOrigin logsconfig.IntakeOrigin) (*logsconfig.Endpoints, error) {
|
||||
func BuildEndpointsWithConfig(endpointPrefix string, intakeTrackType logsconfig.IntakeTrackType, intakeProtocol logsconfig.IntakeProtocol, intakeOrigin logsconfig.IntakeOrigin) (*logsconfig.Endpoints, error) {
|
||||
logsConfig := coreconfig.Config.Logs
|
||||
|
||||
if logsConfig.SendType == "http" || (bool(httpConnectivity) && !(logsConfig.SendType == "tcp")) {
|
||||
switch logsConfig.SendType {
|
||||
case "http":
|
||||
return BuildHTTPEndpointsWithConfig(endpointPrefix, intakeTrackType, intakeProtocol, intakeOrigin)
|
||||
case "tcp":
|
||||
return buildTCPEndpoints(logsConfig)
|
||||
case "kafka":
|
||||
return buildKafkaEndpoints(logsConfig)
|
||||
|
||||
}
|
||||
return buildTCPEndpoints(logsConfig)
|
||||
}
|
||||
|
||||
func buildKafkaEndpoints(logsConfig coreconfig.Logs) (*logsconfig.Endpoints, error) {
|
||||
// return nil, nil
|
||||
// Provide default values for legacy settings when the configuration key does not exist
|
||||
defaultTLS := coreconfig.Config.Logs.SendWithTLS
|
||||
|
||||
main := logsconfig.Endpoint{
|
||||
APIKey: strings.TrimSpace(logsConfig.APIKey),
|
||||
UseCompression: logsConfig.UseCompression,
|
||||
CompressionLevel: logsConfig.CompressionLevel,
|
||||
ConnectionResetInterval: 0,
|
||||
BackoffBase: 1.0,
|
||||
BackoffMax: 120.0,
|
||||
BackoffFactor: 2.0,
|
||||
RecoveryInterval: 2,
|
||||
RecoveryReset: false,
|
||||
Addr: logsConfig.SendTo,
|
||||
Topic: logsConfig.Topic,
|
||||
}
|
||||
|
||||
if intakeTrackType != "" {
|
||||
main.Version = logsconfig.EPIntakeVersion2
|
||||
main.TrackType = intakeTrackType
|
||||
} else {
|
||||
main.Version = logsconfig.EPIntakeVersion1
|
||||
}
|
||||
|
||||
if len(logsConfig.SendTo) != 0 {
|
||||
brokers := strings.Split(logsConfig.SendTo, ",")
|
||||
if len(brokers) == 0 {
|
||||
return nil, fmt.Errorf("wrong send_to content %s", logsConfig.SendTo)
|
||||
}
|
||||
host, port, err := parseAddress(brokers[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse %s: %v", logsConfig.SendTo, err)
|
||||
}
|
||||
main.Host = host
|
||||
main.Port = port
|
||||
main.UseSSL = defaultTLS
|
||||
} else {
|
||||
return nil, fmt.Errorf("empty send_to is not allowed when send_type is kafka")
|
||||
}
|
||||
return NewEndpoints(main, false, "kafka"), nil
|
||||
}
|
||||
|
||||
func buildTCPEndpoints(logsConfig coreconfig.Logs) (*logsconfig.Endpoints, error) {
|
||||
main := logsconfig.Endpoint{
|
||||
APIKey: logsConfig.APIKey,
|
||||
|
|
@ -58,7 +108,7 @@ func buildTCPEndpoints(logsConfig coreconfig.Logs) (*logsconfig.Endpoints, error
|
|||
main.UseSSL = logsConfig.SendWithTLS
|
||||
}
|
||||
|
||||
return NewEndpoints(main, false, false), nil
|
||||
return NewEndpoints(main, false, "tcp"), nil
|
||||
}
|
||||
|
||||
// BuildHTTPEndpoints returns the HTTP endpoints to send logs to.
|
||||
|
|
@ -112,7 +162,7 @@ func BuildHTTPEndpointsWithConfig(endpointPrefix string, intakeTrackType logscon
|
|||
batchMaxSize := 100
|
||||
batchMaxContentSize := 1000000
|
||||
|
||||
return NewEndpointsWithBatchSettings(main, false, true, batchWait, batchMaxConcurrentSend, batchMaxSize, batchMaxContentSize), nil
|
||||
return NewEndpointsWithBatchSettings(main, false, "http", batchWait, batchMaxConcurrentSend, batchMaxSize, batchMaxContentSize), nil
|
||||
}
|
||||
|
||||
// parseAddress returns the host and the port of the address.
|
||||
|
|
@ -129,13 +179,13 @@ func parseAddress(address string) (string, int, error) {
|
|||
}
|
||||
|
||||
// NewEndpoints returns a new endpoints composite with default batching settings
|
||||
func NewEndpoints(main logsconfig.Endpoint, useProto bool, useHTTP bool) *logsconfig.Endpoints {
|
||||
func NewEndpoints(main logsconfig.Endpoint, useProto bool, typ string) *logsconfig.Endpoints {
|
||||
logsConfig := coreconfig.Config.Logs
|
||||
return &logsconfig.Endpoints{
|
||||
Main: main,
|
||||
Additionals: nil,
|
||||
UseProto: useProto,
|
||||
UseHTTP: useHTTP,
|
||||
Type: typ,
|
||||
BatchWait: time.Duration(logsConfig.BatchWait) * time.Second,
|
||||
// TODO support custom param
|
||||
BatchMaxConcurrentSend: 0,
|
||||
|
|
@ -145,12 +195,12 @@ func NewEndpoints(main logsconfig.Endpoint, useProto bool, useHTTP bool) *logsco
|
|||
}
|
||||
|
||||
// NewEndpointsWithBatchSettings returns a new endpoints composite with non-default batching settings specified
|
||||
func NewEndpointsWithBatchSettings(main logsconfig.Endpoint, useProto bool, useHTTP bool, batchWait time.Duration, batchMaxConcurrentSend int, batchMaxSize int, batchMaxContentSize int) *logsconfig.Endpoints {
|
||||
func NewEndpointsWithBatchSettings(main logsconfig.Endpoint, useProto bool, typ string, batchWait time.Duration, batchMaxConcurrentSend int, batchMaxSize int, batchMaxContentSize int) *logsconfig.Endpoints {
|
||||
return &logsconfig.Endpoints{
|
||||
Main: main,
|
||||
Additionals: nil,
|
||||
UseProto: useProto,
|
||||
UseHTTP: useHTTP,
|
||||
Type: typ,
|
||||
BatchWait: batchWait,
|
||||
BatchMaxConcurrentSend: batchMaxConcurrentSend,
|
||||
BatchMaxSize: batchMaxSize,
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
api_key = "ef4ahfbwzwwtlwfpbertgq1i6mq0ab1q"
|
||||
## 是否开启日志采集
|
||||
enable = false
|
||||
## 接受日志的server地址
|
||||
## 接受日志的server地址, http/tcp
|
||||
send_to = "127.0.0.1:17878"
|
||||
## 发送日志的协议 http/tcp
|
||||
## 发送日志的协议 http/tcp/kafka
|
||||
send_type = "http"
|
||||
## 是否压缩发送
|
||||
use_compress = false
|
||||
|
|
@ -33,3 +33,7 @@ collect_container_all = true
|
|||
path = "/opt/tomcat/logs/*.txt"
|
||||
source = "tomcat"
|
||||
service = "my_service"
|
||||
[logs.itmes.output]
|
||||
# kafka topic for this item
|
||||
# brokers = [""]
|
||||
# topic = ""
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"github.com/Shopify/sarama"
|
||||
|
||||
logsconfig "flashcat.cloud/categraf/config/logs"
|
||||
)
|
||||
|
||||
|
|
@ -26,6 +28,12 @@ type (
|
|||
CollectContainerAll bool `json:"collect_container_all" toml:"collect_container_all"`
|
||||
GlobalProcessingRules []*logsconfig.ProcessingRule `json:"processing_rules" toml:"processing_rules"`
|
||||
Items []*logsconfig.LogsConfig `json:"items" toml:"items"`
|
||||
KafkaConfig
|
||||
}
|
||||
KafkaConfig struct {
|
||||
Topic string `json:"topic" toml:"topic"`
|
||||
Brokers []string `json:"brokers" toml:"brokers"`
|
||||
*sarama.Config
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,9 +29,11 @@ const (
|
|||
EPIntakeVersion2
|
||||
)
|
||||
|
||||
// Endpoint holds all the organization and network parameters to send logs to Datadog.
|
||||
// Endpoint holds all the organization and network parameters to send logs
|
||||
type Endpoint struct {
|
||||
APIKey string `mapstructure:"api_key" json:"api_key"`
|
||||
Addr string
|
||||
Topic string
|
||||
Host string
|
||||
Port int
|
||||
UseSSL bool
|
||||
|
|
@ -57,7 +59,7 @@ type Endpoints struct {
|
|||
Main Endpoint
|
||||
Additionals []Endpoint
|
||||
UseProto bool
|
||||
UseHTTP bool
|
||||
Type string
|
||||
BatchWait time.Duration
|
||||
BatchMaxConcurrentSend int
|
||||
BatchMaxSize int
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ package logs
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/Shopify/sarama"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
|
@ -29,45 +30,54 @@ const (
|
|||
|
||||
// LogsConfig represents a log source config, which can be for instance
|
||||
// a file to tail or a port to listen to.
|
||||
type LogsConfig struct {
|
||||
Type string
|
||||
type (
|
||||
LogsConfig struct {
|
||||
Type string
|
||||
|
||||
Port int // Network
|
||||
IdleTimeout string `mapstructure:"idle_timeout" json:"idle_timeout"` // Network
|
||||
Path string // File, Journald
|
||||
Port int // Network
|
||||
IdleTimeout string `mapstructure:"idle_timeout" json:"idle_timeout"` // Network
|
||||
Path string // File, Journald
|
||||
|
||||
Encoding string `mapstructure:"encoding" json:"encoding"` // File
|
||||
ExcludePaths []string `mapstructure:"exclude_paths" json:"exclude_paths"` // File
|
||||
TailingMode string `mapstructure:"start_position" json:"start_position"` // File
|
||||
Encoding string `mapstructure:"encoding" json:"encoding"` // File
|
||||
ExcludePaths []string `mapstructure:"exclude_paths" json:"exclude_paths"` // File
|
||||
TailingMode string `mapstructure:"start_position" json:"start_position"` // File
|
||||
|
||||
IncludeUnits []string `mapstructure:"include_units" json:"include_units"` // Journald
|
||||
ExcludeUnits []string `mapstructure:"exclude_units" json:"exclude_units"` // Journald
|
||||
ContainerMode bool `mapstructure:"container_mode" json:"container_mode"` // Journald
|
||||
IncludeUnits []string `mapstructure:"include_units" json:"include_units"` // Journald
|
||||
ExcludeUnits []string `mapstructure:"exclude_units" json:"exclude_units"` // Journald
|
||||
ContainerMode bool `mapstructure:"container_mode" json:"container_mode"` // Journald
|
||||
|
||||
Image string // Docker
|
||||
Label string // Docker
|
||||
// Name contains the container name
|
||||
Name string // Docker
|
||||
// Identifier contains the container ID
|
||||
Identifier string // Docker
|
||||
Image string // Docker
|
||||
Label string // Docker
|
||||
// Name contains the container name
|
||||
Name string // Docker
|
||||
// Identifier contains the container ID
|
||||
Identifier string // Docker
|
||||
|
||||
ChannelPath string `mapstructure:"channel_path" json:"channel_path"` // Windows Event
|
||||
Query string // Windows Event
|
||||
ChannelPath string `mapstructure:"channel_path" json:"channel_path"` // Windows Event
|
||||
Query string // Windows Event
|
||||
|
||||
// used as input only by the Channel tailer.
|
||||
// could have been unidirectional but the tailer could not close it in this case.
|
||||
Channel chan *ChannelMessage `json:"-"`
|
||||
// used as input only by the Channel tailer.
|
||||
// could have been unidirectional but the tailer could not close it in this case.
|
||||
Channel chan *ChannelMessage `json:"-"`
|
||||
|
||||
Service string
|
||||
Source string
|
||||
SourceCategory string
|
||||
Tags []string
|
||||
ProcessingRules []*ProcessingRule `mapstructure:"log_processing_rules" json:"log_processing_rules"`
|
||||
Service string
|
||||
Source string
|
||||
SourceCategory string
|
||||
Tags []string
|
||||
ProcessingRules []*ProcessingRule `mapstructure:"log_processing_rules" json:"log_processing_rules"`
|
||||
|
||||
AutoMultiLine bool `mapstructure:"auto_multi_line_detection" json:"auto_multi_line_detection"`
|
||||
AutoMultiLineSampleSize int `mapstructure:"auto_multi_line_sample_size" json:"auto_multi_line_sample_size"`
|
||||
AutoMultiLineMatchThreshold float64 `mapstructure:"auto_multi_line_match_threshold" json:"auto_multi_line_match_threshold"`
|
||||
}
|
||||
AutoMultiLine bool `mapstructure:"auto_multi_line_detection" json:"auto_multi_line_detection"`
|
||||
AutoMultiLineSampleSize int `mapstructure:"auto_multi_line_sample_size" json:"auto_multi_line_sample_size"`
|
||||
AutoMultiLineMatchThreshold float64 `mapstructure:"auto_multi_line_match_threshold" json:"auto_multi_line_match_threshold"`
|
||||
|
||||
Kafka KafkaConfig `json:"kafka" toml:"kafka"`
|
||||
}
|
||||
KafkaConfig struct {
|
||||
Brokers []string `json:"brokers" toml:"brokers"`
|
||||
Topic string `json:"topic" toml:"brokers"`
|
||||
*sarama.Config
|
||||
}
|
||||
)
|
||||
|
||||
// TailingMode type
|
||||
type TailingMode uint8
|
||||
|
|
|
|||
19
go.mod
19
go.mod
|
|
@ -4,6 +4,7 @@ go 1.18
|
|||
|
||||
require (
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.0.15
|
||||
github.com/Shopify/sarama v1.34.0
|
||||
github.com/chai2010/winsvc v0.0.0-20200705094454-db7ec320025c
|
||||
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
|
||||
github.com/docker/docker v20.10.16+incompatible
|
||||
|
|
@ -31,7 +32,7 @@ require (
|
|||
github.com/stretchr/testify v1.7.2
|
||||
github.com/toolkits/pkg v1.3.0
|
||||
github.com/ulricqin/gosnmp v0.0.1
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5
|
||||
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2
|
||||
golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32
|
||||
golang.org/x/text v0.3.7
|
||||
)
|
||||
|
|
@ -47,6 +48,9 @@ require (
|
|||
github.com/docker/distribution v2.8.1+incompatible // indirect
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/eapache/go-resiliency v1.2.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
github.com/fatih/camelcase v1.0.0 // indirect
|
||||
github.com/fatih/color v1.9.0 // indirect
|
||||
github.com/fatih/structs v1.1.0 // indirect
|
||||
|
|
@ -57,12 +61,21 @@ require (
|
|||
github.com/godror/knownpb v0.1.0 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
|
||||
github.com/hashicorp/go-hclog v0.12.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.1 // indirect
|
||||
github.com/hashicorp/serf v0.9.6 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/gofork v1.0.0 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/klauspost/compress v1.15.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.6 // indirect
|
||||
github.com/mattn/go-isatty v0.0.12 // indirect
|
||||
|
|
@ -77,8 +90,9 @@ require (
|
|||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/shopspring/decimal v1.3.1 // indirect
|
||||
github.com/sirupsen/logrus v1.7.0 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/stretchr/objx v0.1.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.10 // indirect
|
||||
github.com/tklauser/numcpus v0.4.0 // indirect
|
||||
|
|
@ -86,6 +100,7 @@ require (
|
|||
go.opentelemetry.io/otel v1.7.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.7.0 // indirect
|
||||
go.uber.org/automaxprocs v1.4.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect
|
||||
google.golang.org/grpc v1.33.1 // indirect
|
||||
|
|
|
|||
70
go.sum
70
go.sum
|
|
@ -42,6 +42,10 @@ github.com/ClickHouse/clickhouse-go/v2 v2.0.15 h1:lLAZliqrZEygkxosLaW1qHyeTb4Ho7
|
|||
github.com/ClickHouse/clickhouse-go/v2 v2.0.15/go.mod h1:Z21o82zD8FFqefOQDg93c0XITlxGbTsWQuRm588Azkk=
|
||||
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
github.com/Shopify/sarama v1.34.0 h1:j4zTaFHFnfvuV2fdLZyXqIg0Tu4Mzl9f064Z5/H+o4o=
|
||||
github.com/Shopify/sarama v1.34.0/go.mod h1:V2ceE9UupUf4/oP1Z38SI49fAnD0/MtkqDDHvolIeeQ=
|
||||
github.com/Shopify/toxiproxy/v2 v2.3.0 h1:62YkpiP4bzdhKMH+6uC5E95y608k3zDwdzuBMsnn3uQ=
|
||||
github.com/Shopify/toxiproxy/v2 v2.3.0/go.mod h1:KvQTtB6RjCJY4zqNJn7C7JDFgsG5uoHYDirfUfpIm0c=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
|
|
@ -75,6 +79,7 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h
|
|||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
|
||||
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -90,6 +95,12 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
|
|||
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
|
||||
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
|
||||
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
|
||||
github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
|
|
@ -101,6 +112,8 @@ github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
|
|||
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
|
||||
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
|
||||
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
|
||||
github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
|
||||
github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
|
||||
github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk=
|
||||
|
|
@ -217,6 +230,9 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
|
|||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
|
|
@ -235,16 +251,18 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
|
|||
github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4=
|
||||
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
|
||||
github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
|
|
@ -262,6 +280,18 @@ github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod
|
|||
github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY=
|
||||
github.com/influxdata/line-protocol/v2 v2.2.1 h1:EAPkqJ9Km4uAxtMRgUubJyqAr6zgWM0dznKMLRauQRE=
|
||||
github.com/influxdata/line-protocol/v2 v2.2.1/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
|
||||
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
|
||||
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
||||
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
|
||||
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
|
||||
|
|
@ -279,6 +309,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
|
|||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U=
|
||||
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7 h1:SWlt7BoQNASbhTUD0Oy5yysI2seJ7vWuGUp///OM4TM=
|
||||
github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7/go.mod h1:Y2SaZf2Rzd0pXkLVhLlCiAXFCLSXAIbTKDivVgff/AM=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
|
|
@ -286,8 +318,9 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv
|
|||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
|
|
@ -336,7 +369,6 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
|||
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/oklog/ulid/v2 v2.0.2 h1:r4fFzBm+bv0wNKNh5eXTwU7i85y5x+uwkxCUTNVQqLc=
|
||||
|
|
@ -391,9 +423,14 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
|
|||
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg=
|
||||
github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62/go.mod h1:65XQgovT59RWatovFwnwocoUxiI/eENTnOY5GK3STuY=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
|
|
@ -404,11 +441,13 @@ github.com/shirou/gopsutil/v3 v3.22.3/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H
|
|||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
||||
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
|
||||
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
|
|
@ -417,6 +456,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
|
|
@ -430,6 +471,10 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM
|
|||
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
|
||||
github.com/ulricqin/gosnmp v0.0.1 h1:LmJ4y0tmnIe+qBeBAX9UjQvRLezso7VnN0bgslYnfzI=
|
||||
github.com/ulricqin/gosnmp v0.0.1/go.mod h1:9OasJbP94MjBGOLNghlVwgG3UN05ATurou1Gw+6rdzU=
|
||||
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
|
||||
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
|
|
@ -454,6 +499,9 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
|
|||
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE=
|
||||
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
|
|
@ -518,8 +566,10 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
|
|||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
|
||||
golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y=
|
||||
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -589,12 +639,15 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 h1:Js08h5hqB5xyWR789+QqueR6sDE8mk+YvpETZ+F6X9Y=
|
||||
golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
@ -742,10 +795,12 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
|
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
|
@ -757,6 +812,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
|||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
|
||||
|
|
|
|||
|
|
@ -1,30 +1,3 @@
|
|||
# logs-agent
|
||||
# logs
|
||||
|
||||
logs-agent collects logs and submits them to datadog's infrastructure.
|
||||
|
||||
## Structure
|
||||
|
||||
`logs` reads the config files, and instantiates what's needed.
|
||||
Each log line comes from a source (e.g. file, network, docker), and then enters one of the available _pipeline - tailer|listener|container -> decoder -> processor -> sender -> auditor_
|
||||
|
||||
`Tailer` tails a file and submits data to the processors
|
||||
|
||||
`Listener` listens on local network (TCP, UDP, Unix) and submits data to the processors
|
||||
|
||||
`Container` scans docker logs from stdout/stderr and submits data to the processors
|
||||
|
||||
`Decoder` converts bytes arrays into messages
|
||||
|
||||
`Processor` updates the messages, filtering, redacting or adding metadata, and submits to the forwarder
|
||||
|
||||
`Sender` submits the messages to the intake, and notifies the auditor
|
||||
|
||||
`Auditor` notes that messages were properly submitted, stores offsets for agent restarts
|
||||
|
||||
## Tests
|
||||
|
||||
```
|
||||
# Run the unit tests
|
||||
inv test --targets=./pkg/logs --timeout=10
|
||||
|
||||
```
|
||||
forked from datadog logs-agent
|
||||
|
|
|
|||
|
|
@ -0,0 +1,73 @@
|
|||
// Unless explicitly stated otherwise all files in this repository are licensed
|
||||
// under the Apache License Version 2.0.
|
||||
// This product includes software developed at Datadog (https://www.datadoghq.com/).
|
||||
// Copyright 2016-present Datadog, Inc.
|
||||
|
||||
package kafka
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
)
|
||||
|
||||
// ContentEncoding encodes the payload
|
||||
type ContentEncoding interface {
|
||||
name() string
|
||||
encode(payload []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
// IdentityContentType encodes the payload using the identity function
|
||||
var IdentityContentType ContentEncoding = &identityContentType{}
|
||||
|
||||
type identityContentType struct{}
|
||||
|
||||
func (c *identityContentType) name() string {
|
||||
return "identity"
|
||||
}
|
||||
|
||||
func (c *identityContentType) encode(payload []byte) ([]byte, error) {
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
// GzipContentEncoding encodes the payload using gzip algorithm
|
||||
type GzipContentEncoding struct {
|
||||
level int
|
||||
}
|
||||
|
||||
// NewGzipContentEncoding creates a new Gzip content type
|
||||
func NewGzipContentEncoding(level int) *GzipContentEncoding {
|
||||
if level < gzip.NoCompression {
|
||||
level = gzip.NoCompression
|
||||
} else if level > gzip.BestCompression {
|
||||
level = gzip.BestCompression
|
||||
}
|
||||
|
||||
return &GzipContentEncoding{
|
||||
level,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *GzipContentEncoding) name() string {
|
||||
return "gzip"
|
||||
}
|
||||
|
||||
func (c *GzipContentEncoding) encode(payload []byte) ([]byte, error) {
|
||||
var compressedPayload bytes.Buffer
|
||||
gzipWriter, err := gzip.NewWriterLevel(&compressedPayload, c.level)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, err = gzipWriter.Write(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = gzipWriter.Flush()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = gzipWriter.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return compressedPayload.Bytes(), nil
|
||||
}
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
package kafka
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
coreconfig "flashcat.cloud/categraf/config"
|
||||
"github.com/Shopify/sarama"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
logsconfig "flashcat.cloud/categraf/config/logs"
|
||||
"flashcat.cloud/categraf/logs/client"
|
||||
"flashcat.cloud/categraf/pkg/backoff"
|
||||
)
|
||||
|
||||
// ContentType options,
|
||||
const (
|
||||
TextContentType = "text/plain"
|
||||
JSONContentType = "application/json"
|
||||
)
|
||||
|
||||
// HTTP errors.
|
||||
var (
|
||||
errClient = errors.New("client error")
|
||||
errServer = errors.New("server error")
|
||||
)
|
||||
|
||||
// emptyPayload is an empty payload used to check HTTP connectivity without sending logs.
|
||||
var emptyPayload []byte
|
||||
|
||||
// Destination sends a payload over HTTP.
|
||||
type Destination struct {
|
||||
topic string
|
||||
brokers []string
|
||||
|
||||
apiKey string
|
||||
contentType string
|
||||
contentEncoding ContentEncoding
|
||||
client sarama.SyncProducer
|
||||
destinationsContext *client.DestinationsContext
|
||||
once sync.Once
|
||||
payloadChan chan []byte
|
||||
climit chan struct{} // semaphore for limiting concurrent background sends
|
||||
backoff backoff.Policy
|
||||
nbErrors int
|
||||
blockedUntil time.Time
|
||||
protocol logsconfig.IntakeProtocol
|
||||
origin logsconfig.IntakeOrigin
|
||||
}
|
||||
|
||||
// NewDestination returns a new Destination.
|
||||
// If `maxConcurrentBackgroundSends` > 0, then at most that many background payloads will be sent concurrently, else
|
||||
// there is no concurrency and the background sending pipeline will block while sending each payload.
|
||||
// TODO: add support for SOCKS5
|
||||
func NewDestination(endpoint logsconfig.Endpoint, contentType string, destinationsContext *client.DestinationsContext, maxConcurrentBackgroundSends int) *Destination {
|
||||
return newDestination(endpoint, contentType, destinationsContext, time.Second*10, maxConcurrentBackgroundSends)
|
||||
}
|
||||
|
||||
func newDestination(endpoint logsconfig.Endpoint, contentType string, destinationsContext *client.DestinationsContext, timeout time.Duration, maxConcurrentBackgroundSends int) *Destination {
|
||||
if maxConcurrentBackgroundSends < 0 {
|
||||
maxConcurrentBackgroundSends = 0
|
||||
}
|
||||
|
||||
policy := backoff.NewPolicy(
|
||||
endpoint.BackoffFactor,
|
||||
endpoint.BackoffBase,
|
||||
endpoint.BackoffMax,
|
||||
endpoint.RecoveryInterval,
|
||||
endpoint.RecoveryReset,
|
||||
)
|
||||
brokers := strings.Split(endpoint.Addr, ",")
|
||||
c, err := sarama.NewSyncProducer(brokers, coreconfig.Config.Logs.Config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &Destination{
|
||||
topic: endpoint.Topic,
|
||||
brokers: brokers,
|
||||
apiKey: endpoint.APIKey,
|
||||
contentType: contentType,
|
||||
contentEncoding: buildContentEncoding(endpoint),
|
||||
client: c,
|
||||
destinationsContext: destinationsContext,
|
||||
climit: make(chan struct{}, maxConcurrentBackgroundSends),
|
||||
backoff: policy,
|
||||
protocol: endpoint.Protocol,
|
||||
origin: endpoint.Origin,
|
||||
}
|
||||
}
|
||||
|
||||
func errorToTag(err error) string {
|
||||
if err == nil {
|
||||
return "none"
|
||||
} else if _, ok := err.(*client.RetryableError); ok {
|
||||
return "retryable"
|
||||
} else {
|
||||
return "non-retryable"
|
||||
}
|
||||
}
|
||||
|
||||
// Send sends a payload over HTTP,
|
||||
// the error returned can be retryable and it is the responsibility of the callee to retry.
|
||||
func (d *Destination) Send(payload []byte) error {
|
||||
if d.blockedUntil.After(time.Now()) {
|
||||
// log.Printf("%s: sleeping until %v before retrying\n", d.url, d.blockedUntil)
|
||||
d.waitForBackoff()
|
||||
}
|
||||
|
||||
err := d.unconditionalSend(payload)
|
||||
|
||||
if _, ok := err.(*client.RetryableError); ok {
|
||||
d.nbErrors = d.backoff.IncError(d.nbErrors)
|
||||
} else {
|
||||
d.nbErrors = d.backoff.DecError(d.nbErrors)
|
||||
}
|
||||
|
||||
d.blockedUntil = time.Now().Add(d.backoff.GetBackoffDuration(d.nbErrors))
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *Destination) unconditionalSend(payload []byte) (err error) {
|
||||
ctx := d.destinationsContext.Context()
|
||||
|
||||
encodedPayload, err := d.contentEncoding.encode(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = NewBuilder().WithMessage(d.apiKey, encodedPayload).WithTopic(d.topic).Send(d.client)
|
||||
if err != nil {
|
||||
if ctx.Err() == context.Canceled {
|
||||
return ctx.Err()
|
||||
}
|
||||
// most likely a network or a connect error, the callee should retry.
|
||||
return client.NewRetryableError(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SendAsync sends a payload in background.
|
||||
func (d *Destination) SendAsync(payload []byte) {
|
||||
d.once.Do(func() {
|
||||
payloadChan := make(chan []byte, logsconfig.ChanSize)
|
||||
d.sendInBackground(payloadChan)
|
||||
d.payloadChan = payloadChan
|
||||
})
|
||||
d.payloadChan <- payload
|
||||
}
|
||||
|
||||
// sendInBackground sends all payloads from payloadChan in background.
|
||||
func (d *Destination) sendInBackground(payloadChan chan []byte) {
|
||||
ctx := d.destinationsContext.Context()
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case payload := <-payloadChan:
|
||||
// if the channel is non-buffered then there is no concurrency and we block on sending each payload
|
||||
if cap(d.climit) == 0 {
|
||||
d.unconditionalSend(payload) //nolint:errcheck
|
||||
break
|
||||
}
|
||||
d.climit <- struct{}{}
|
||||
go func() {
|
||||
d.unconditionalSend(payload) //nolint:errcheck
|
||||
<-d.climit
|
||||
}()
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func buildContentEncoding(endpoint logsconfig.Endpoint) ContentEncoding {
|
||||
if endpoint.UseCompression {
|
||||
return NewGzipContentEncoding(endpoint.CompressionLevel)
|
||||
}
|
||||
return IdentityContentType
|
||||
}
|
||||
|
||||
func (d *Destination) waitForBackoff() {
|
||||
ctx, cancel := context.WithDeadline(d.destinationsContext.Context(), d.blockedUntil)
|
||||
defer cancel()
|
||||
<-ctx.Done()
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
package kafka
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/Shopify/sarama"
|
||||
)
|
||||
|
||||
type MessageBuilder struct {
|
||||
sarama.ProducerMessage
|
||||
}
|
||||
|
||||
func NewBuilder() *MessageBuilder {
|
||||
return &MessageBuilder{}
|
||||
}
|
||||
|
||||
func (m *MessageBuilder) WithMessage(key string, value []byte) *MessageBuilder {
|
||||
m.Key = sarama.StringEncoder(key)
|
||||
m.Value = sarama.ByteEncoder(value)
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *MessageBuilder) WithTopic(topic string) *MessageBuilder {
|
||||
m.Topic = topic
|
||||
return m
|
||||
}
|
||||
|
||||
func (s *MessageBuilder) build() (*sarama.ProducerMessage, error) {
|
||||
switch {
|
||||
case len(s.Topic) == 0:
|
||||
return nil, fmt.Errorf("Message (%s) must not be nil", "topic")
|
||||
case s.Key.Length() == 0:
|
||||
return nil, fmt.Errorf("Message (%s) must not be nil", "key")
|
||||
case s.Value.Length() == 0:
|
||||
return nil, fmt.Errorf("Message (%s) must not be nil", "value")
|
||||
}
|
||||
return &s.ProducerMessage, nil
|
||||
}
|
||||
|
||||
func (m *MessageBuilder) Send(producer sarama.SyncProducer) error {
|
||||
if producer == nil {
|
||||
return fmt.Errorf("empty producer")
|
||||
}
|
||||
|
||||
msg, err := m.build()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, _, err = producer.SendMessage(msg)
|
||||
return err
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ import (
|
|||
logsconfig "flashcat.cloud/categraf/config/logs"
|
||||
"flashcat.cloud/categraf/logs/client"
|
||||
"flashcat.cloud/categraf/logs/client/http"
|
||||
"flashcat.cloud/categraf/logs/client/kafka"
|
||||
"flashcat.cloud/categraf/logs/client/tcp"
|
||||
"flashcat.cloud/categraf/logs/diagnostic"
|
||||
"flashcat.cloud/categraf/logs/message"
|
||||
|
|
@ -27,42 +28,46 @@ type Pipeline struct {
|
|||
|
||||
// NewPipeline returns a new Pipeline
|
||||
func NewPipeline(outputChan chan *message.Message, processingRules []*logsconfig.ProcessingRule, endpoints *logsconfig.Endpoints, destinationsContext *client.DestinationsContext, diagnosticMessageReceiver diagnostic.MessageReceiver, serverless bool) *Pipeline {
|
||||
var destinations *client.Destinations
|
||||
if endpoints.UseHTTP {
|
||||
var (
|
||||
destinations *client.Destinations
|
||||
strategy sender.Strategy
|
||||
encoder processor.Encoder
|
||||
)
|
||||
switch endpoints.Type {
|
||||
case "http":
|
||||
main := http.NewDestination(endpoints.Main, http.JSONContentType, destinationsContext, endpoints.BatchMaxConcurrentSend)
|
||||
additionals := []client.Destination{}
|
||||
for _, endpoint := range endpoints.Additionals {
|
||||
additionals = append(additionals, http.NewDestination(endpoint, http.JSONContentType, destinationsContext, endpoints.BatchMaxConcurrentSend))
|
||||
}
|
||||
destinations = client.NewDestinations(main, additionals)
|
||||
} else {
|
||||
strategy = sender.NewBatchStrategy(sender.ArraySerializer, endpoints.BatchWait, endpoints.BatchMaxConcurrentSend, endpoints.BatchMaxSize, endpoints.BatchMaxContentSize, "logs")
|
||||
encoder = processor.JSONEncoder
|
||||
case "kafka":
|
||||
main := kafka.NewDestination(endpoints.Main, http.JSONContentType, destinationsContext, endpoints.BatchMaxConcurrentSend)
|
||||
additionals := []client.Destination{}
|
||||
for _, endpoint := range endpoints.Additionals {
|
||||
additionals = append(additionals, kafka.NewDestination(endpoint, http.JSONContentType, destinationsContext, endpoints.BatchMaxConcurrentSend))
|
||||
}
|
||||
destinations = client.NewDestinations(main, additionals)
|
||||
strategy = sender.StreamStrategy
|
||||
encoder = processor.JSONEncoder
|
||||
case "tcp":
|
||||
main := tcp.NewDestination(endpoints.Main, endpoints.UseProto, destinationsContext)
|
||||
additionals := []client.Destination{}
|
||||
for _, endpoint := range endpoints.Additionals {
|
||||
additionals = append(additionals, tcp.NewDestination(endpoint, endpoints.UseProto, destinationsContext))
|
||||
}
|
||||
destinations = client.NewDestinations(main, additionals)
|
||||
strategy = sender.StreamStrategy
|
||||
encoder = processor.RawEncoder
|
||||
}
|
||||
|
||||
senderChan := make(chan *message.Message, logsconfig.ChanSize)
|
||||
|
||||
var strategy sender.Strategy
|
||||
if endpoints.UseHTTP || serverless {
|
||||
strategy = sender.NewBatchStrategy(sender.ArraySerializer, endpoints.BatchWait, endpoints.BatchMaxConcurrentSend, endpoints.BatchMaxSize, endpoints.BatchMaxContentSize, "logs")
|
||||
} else {
|
||||
strategy = sender.StreamStrategy
|
||||
}
|
||||
sender := sender.NewSender(senderChan, outputChan, destinations, strategy)
|
||||
|
||||
var encoder processor.Encoder
|
||||
if serverless {
|
||||
encoder = processor.JSONServerlessEncoder
|
||||
} else if endpoints.UseHTTP {
|
||||
encoder = processor.JSONEncoder
|
||||
} else if endpoints.UseProto {
|
||||
if endpoints.UseProto {
|
||||
encoder = processor.ProtoEncoder
|
||||
} else {
|
||||
encoder = processor.RawEncoder
|
||||
}
|
||||
|
||||
inputChan := make(chan *message.Message, logsconfig.ChanSize)
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ func (b *Builder) BuildStatus() Status {
|
|||
StatusMetrics: b.getMetricsStatus(),
|
||||
Warnings: b.getWarnings(),
|
||||
Errors: b.getErrors(),
|
||||
UseHTTP: b.getUseHTTP(),
|
||||
Type: b.getType(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -57,8 +57,8 @@ func (b *Builder) getIsRunning() bool {
|
|||
return atomic.LoadInt32(b.isRunning) != 0
|
||||
}
|
||||
|
||||
func (b *Builder) getUseHTTP() bool {
|
||||
return b.endpoints.UseHTTP
|
||||
func (b *Builder) getType() string {
|
||||
return b.endpoints.Type
|
||||
}
|
||||
|
||||
func (b *Builder) getEndpoints() []string {
|
||||
|
|
@ -80,7 +80,8 @@ func (b *Builder) formatEndpoint(endpoint logsconfig.Endpoint, prefix string) st
|
|||
port := endpoint.Port
|
||||
|
||||
var protocol string
|
||||
if b.endpoints.UseHTTP {
|
||||
switch b.endpoints.Type {
|
||||
case "http":
|
||||
if endpoint.UseSSL {
|
||||
protocol = "HTTPS"
|
||||
if port == 0 {
|
||||
|
|
@ -95,7 +96,13 @@ func (b *Builder) formatEndpoint(endpoint logsconfig.Endpoint, prefix string) st
|
|||
port = 80 // use default port
|
||||
}
|
||||
}
|
||||
} else {
|
||||
case "kafka":
|
||||
if endpoint.UseSSL {
|
||||
protocol = " SSL Encrypted TCP (to Kafka)"
|
||||
} else {
|
||||
protocol = "TCP (to Kafka)"
|
||||
}
|
||||
case "tcp":
|
||||
if endpoint.UseSSL {
|
||||
protocol = "SSL encrypted TCP"
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ type Status struct {
|
|||
Integrations []Integration `json:"integrations"`
|
||||
Errors []string `json:"errors"`
|
||||
Warnings []string `json:"warnings"`
|
||||
UseHTTP bool `json:"use_http"`
|
||||
Type string `json:"use_http"`
|
||||
}
|
||||
|
||||
// Init instantiates the builder that builds the status on the fly.
|
||||
|
|
|
|||
Loading…
Reference in New Issue