logs config

config/config.go

@@ -54,6 +54,7 @@ type ConfigType struct {
 	Global    Global         `toml:"global"`
 	WriterOpt WriterOpt      `toml:"writer_opt"`
 	Writers   []WriterOption `toml:"writers"`
+	Logs      Logs           `toml:"logs"`
 }
 
 var Config *ConfigType

config/logs.go

@@ -1 +1,70 @@
 package config
+
+import (
+	logsconfig "flashcat.cloud/categraf/config/logs"
+)
+
+const (
+	Docker     = "docker"
+	Kubernetes = "kubernetes"
+)
+
+type Logs struct {
+	APIKey                string                       `toml:"api_key"`
+	Enable                bool                         `toml:"enable"`
+	SendTo                string                       `toml:"send_to"`
+	SendType              string                       `toml:"send_type"`
+	UseCompression        bool                         `toml:"use_compression"`
+	CompressionLevel      int                          `toml:"compression_level"`
+	SendWithTLS           bool                         `toml:"send_with_tls"`
+	BatchWait             int                          `toml:"batch_wait"`
+	RunPath               string                       `toml:"run_path"`
+	OpenFilesLimit        int                          `toml:"open_files_limit"`
+	ScanPeriod            int                          `toml:"scan_period"`
+	FrameSize             int                          `toml:"frame_size"`
+	CollectContainerAll   bool                         `toml:"container_collect_all"`
+	GlobalProcessingRules []*logsconfig.ProcessingRule `toml:"processing_rules"`
+}
+
+func GetLogRunPath() string {
+	if len(Config.Logs.RunPath) == 0 {
+		Config.Logs.RunPath = "/opt/categraf/run"
+	}
+	return Config.Logs.RunPath
+}
+func GetLogReadTimeout() int {
+	return 30
+}
+
+func OpenLogsLimit() int {
+	if Config.Logs.OpenFilesLimit == 0 {
+		Config.Logs.OpenFilesLimit = 100
+	}
+	return Config.Logs.OpenFilesLimit
+}
+
+func FileScanPeriod() int {
+	if Config.Logs.ScanPeriod == 0 {
+		Config.Logs.ScanPeriod = 10
+	}
+	return Config.Logs.ScanPeriod
+}
+
+func LogFrameSize() int {
+	if Config.Logs.FrameSize == 0 {
+		Config.Logs.FrameSize = 9000
+	}
+	return Config.Logs.FrameSize
+}
+
+func ValidatePodContainerID() bool {
+	return false
+}
+
+func IsFeaturePresent(t string) bool {
+	return false
+}
+
+func GetContainerCollectAll() bool {
+	return Config.Logs.CollectContainerAll
+}

go.mod

@@ -3,11 +3,13 @@ module flashcat.cloud/categraf
 go 1.17
 
 require (
+	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/go-ping/ping v0.0.0-20211130115550-779d1e919534
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gobwas/glob v0.2.3
 	github.com/godror/godror v0.33.0
+	github.com/gogo/protobuf v1.1.1
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4
 	github.com/influxdata/line-protocol/v2 v2.2.1
@@ -15,18 +17,23 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7
 	github.com/matttproud/golang_protobuf_extensions v1.0.1
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.32.1
 	github.com/prometheus/prometheus v2.5.0+incompatible
 	github.com/shirou/gopsutil/v3 v3.22.3
+	github.com/stretchr/testify v1.7.1
 	github.com/toolkits/pkg v1.2.9
+	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
+	golang.org/x/text v0.3.7
 )
 
 require (
 	github.com/BurntSushi/toml v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
@@ -34,20 +41,20 @@ require (
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/godror/knownpb v0.1.0 // indirect
-	github.com/gogo/protobuf v1.1.1 // indirect
 	github.com/google/uuid v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/stretchr/objx v0.1.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.uber.org/automaxprocs v1.4.0 // indirect
-	golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect
 	google.golang.org/grpc v1.33.1 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
 )

go.sum

@@ -53,6 +53,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
+github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -251,6 +253,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
 github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -292,6 +296,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=