p17860254
/
categraf
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

extrace log config

This commit is contained in:
kongfei 2022-06-01 18:15:10 +08:00
parent 9d3598aaf6
commit f840ffdc48
33 changed files with 94 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
agent/agent.go
View File

@ -105,8 +105,8 @@ func (a *Agent) startInputs() error {
// construct input instance
instance := creator()
if config.Config.Logs.Enable {
go startLogAgent(instance)
if config.LogConfig.Enable {
go startLogAgent()
}
// set configurations for input instance
cfg.LoadConfigs(path.Join(config.Config.ConfigDir, "input."+name), instance)

29
agent/logs_agent.go
View File

@ -15,7 +15,6 @@ import (
coreConfig "flashcat.cloud/categraf/config"
logsconfig "flashcat.cloud/categraf/config/logs"
"flashcat.cloud/categraf/inputs"
"flashcat.cloud/categraf/pkg/logs/auditor"
"flashcat.cloud/categraf/pkg/logs/client"
"flashcat.cloud/categraf/pkg/logs/client/http"
@ -148,24 +147,22 @@ const (
invalidProcessingRules = "invalid_global_processing_rules"
)
func startLogAgent(instance inputs.Input) {
func startLogAgent() {
logSources := logsconfig.NewLogSources()
var sources []*logsconfig.LogSource
if coreConfig.Config.Logs.Enable {
for _, c := range instance.LogsConfig() {
if c == nil {
continue
}
source := logsconfig.NewLogSource(c.Name, c)
sources = append(sources, source)
if err := c.Validate(); err != nil {
log.Println("W! Invalid logs configuration:", err)
source.Status.Error(err)
continue
}
logSources.AddSource(source)
for _, c := range coreConfig.LogConfig.Items {
if c == nil {
continue
}
source := logsconfig.NewLogSource(c.Name, c)
sources = append(sources, source)
if err := c.Validate(); err != nil {
log.Println("W! Invalid logs configuration:", err)
source.Status.Error(err)
continue
}
logSources.AddSource(source)
}
if len(sources) == 0 {
@ -202,7 +199,7 @@ func GetContainerColloectAll() bool {
// GlobalProcessingRules returns the global processing rules to apply to all logs.
func GlobalProcessingRules() ([]*logsconfig.ProcessingRule, error) {
rules := coreConfig.Config.Logs.GlobalProcessingRules
rules := coreConfig.LogConfig.GlobalProcessingRules
err := logsconfig.ValidateProcessingRules(rules)
if err != nil {
return nil, err

10
agent/logs_endpoints.go
View File

@ -29,12 +29,12 @@ func BuildEndpoints(httpConnectivity logsconfig.HTTPConnectivity, intakeTrackTyp
// BuildEndpointsWithConfig returns the endpoints to send logs.
func BuildEndpointsWithConfig(endpointPrefix string, httpConnectivity logsconfig.HTTPConnectivity, intakeTrackType logsconfig.IntakeTrackType, intakeProtocol logsconfig.IntakeProtocol, intakeOrigin logsconfig.IntakeOrigin) (*logsconfig.Endpoints, error) {
logsConfig := coreconfig.Config.Logs
logsConfig := coreconfig.LogConfig
if logsConfig.SendType == "http" || (bool(httpConnectivity) && !(logsConfig.SendType == "tcp")) {
return BuildHTTPEndpointsWithConfig(endpointPrefix, intakeTrackType, intakeProtocol, intakeOrigin)
}
return buildTCPEndpoints(logsConfig)
return buildTCPEndpoints(*logsConfig)
}
func buildTCPEndpoints(logsConfig coreconfig.Logs) (*logsconfig.Endpoints, error) {
@ -69,8 +69,8 @@ func BuildHTTPEndpoints(intakeTrackType logsconfig.IntakeTrackType, intakeProtoc
// BuildHTTPEndpointsWithConfig uses two arguments that instructs it how to access configuration parameters, then returns the HTTP endpoints to send logs to. This function is able to default to the 'classic' BuildHTTPEndpoints() w ldHTTPEndpointsWithConfigdefault variables logsConfigDefaultKeys and httpEndpointPrefix
func BuildHTTPEndpointsWithConfig(endpointPrefix string, intakeTrackType logsconfig.IntakeTrackType, intakeProtocol logsconfig.IntakeProtocol, intakeOrigin logsconfig.IntakeOrigin) (*logsconfig.Endpoints, error) {
// Provide default values for legacy settings when the configuration key does not exist
logsConfig := coreconfig.Config.Logs
defaultTLS := coreconfig.Config.Logs.SendWithTLS
logsConfig := coreconfig.LogConfig
defaultTLS := coreconfig.LogConfig.SendWithTLS
main := logsconfig.Endpoint{
APIKey: strings.TrimSpace(logsConfig.APIKey),
@ -130,7 +130,7 @@ func parseAddress(address string) (string, int, error) {
// NewEndpoints returns a new endpoints composite with default batching settings
func NewEndpoints(main logsconfig.Endpoint, useProto bool, useHTTP bool) *logsconfig.Endpoints {
logsConfig := coreconfig.Config.Logs
logsConfig := coreconfig.LogConfig
return &logsconfig.Endpoints{
Main: main,
Additionals: nil,

1
config/config.go
View File

@ -47,7 +47,6 @@ type ConfigType struct {
Global Global `toml:"global"`
WriterOpt WriterOpt `toml:"writer_opt"`
Writers []WriterOption `toml:"writers"`
Logs Logs `toml:"logs"`
}
var Config *ConfigType

59
config/logs.go
View File

@ -1,7 +1,14 @@
package config
import (
"encoding/json"
"fmt"
"path"
"github.com/toolkits/pkg/file"
logsconfig "flashcat.cloud/categraf/config/logs"
"flashcat.cloud/categraf/pkg/cfg"
)
const (
@ -22,39 +29,63 @@ type Logs struct {
OpenFilesLimit int `toml:"open_files_limit"`
ScanPeriod int `toml:"scan_period"`
FrameSize int `toml:"frame_size"`
CollectContainerAll bool `toml:"container_collect_all"`
CollectContainerAll bool `toml:"collect_container_all"`
GlobalProcessingRules []*logsconfig.ProcessingRule `toml:"processing_rules"`
Items []*logsconfig.LogsConfig `toml:"items"`
}
var (
LogConfig *Logs
)
func InitLogConfig(configDir string) error {
configFile := path.Join(configDir, "logs.toml")
if !file.IsExist(configFile) {
return fmt.Errorf("configuration file(%s) not found", configFile)
}
LogConfig = &Logs{}
if err := cfg.LoadConfig(configFile, LogConfig); err != nil {
return fmt.Errorf("failed to load configs of dir: %s", configDir)
}
if Config != nil && Config.Global.PrintConfigs {
bs, _ := json.MarshalIndent(LogConfig, "", " ")
fmt.Println(string(bs))
}
return nil
}
func GetLogRunPath() string {
if len(Config.Logs.RunPath) == 0 {
Config.Logs.RunPath = "/opt/categraf/run"
if len(LogConfig.RunPath) == 0 {
LogConfig.RunPath = "/opt/categraf/run"
}
return Config.Logs.RunPath
return LogConfig.RunPath
}
func GetLogReadTimeout() int {
return 30
}
func OpenLogsLimit() int {
if Config.Logs.OpenFilesLimit == 0 {
Config.Logs.OpenFilesLimit = 100
if LogConfig.OpenFilesLimit == 0 {
LogConfig.OpenFilesLimit = 100
}
return Config.Logs.OpenFilesLimit
return LogConfig.OpenFilesLimit
}
func FileScanPeriod() int {
if Config.Logs.ScanPeriod == 0 {
Config.Logs.ScanPeriod = 10
if LogConfig.ScanPeriod == 0 {
LogConfig.ScanPeriod = 10
}
return Config.Logs.ScanPeriod
return LogConfig.ScanPeriod
}
func LogFrameSize() int {
if Config.Logs.FrameSize == 0 {
Config.Logs.FrameSize = 9000
if LogConfig.FrameSize == 0 {
LogConfig.FrameSize = 9000
}
return Config.Logs.FrameSize
return LogConfig.FrameSize
}
func ValidatePodContainerID() bool {
@ -66,5 +97,5 @@ func IsFeaturePresent(t string) bool {
}
func GetContainerCollectAll() bool {
return Config.Logs.CollectContainerAll
return LogConfig.CollectContainerAll
}

9
inputs/cpu/logs.go
View File

@ -1,9 +0,0 @@
package cpu
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (c *CPUStats) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/disk/logs.go
View File

@ -1,9 +0,0 @@
package disk
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (s *DiskStats) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/diskio/logs.go
View File

@ -1,9 +0,0 @@
package diskio
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (d *DiskIO) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/exec/logs.go
View File

@ -1,9 +0,0 @@
package exec
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (e *Exec) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/http_response/logs.go
View File

@ -1,9 +0,0 @@
package http_response
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (c *HTTPResponse) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

2
inputs/inputs.go
View File

@ -2,7 +2,6 @@ package inputs
import (
"flashcat.cloud/categraf/config"
logsconfig "flashcat.cloud/categraf/config/logs"
"flashcat.cloud/categraf/pkg/conv"
"flashcat.cloud/categraf/types"
"github.com/toolkits/pkg/container/list"
@ -14,7 +13,6 @@ type Input interface {
Prefix() string
GetInterval() config.Duration
Gather(slist *list.SafeList)
LogsConfig() []*logsconfig.LogsConfig
}
type Creator func() Input

9
inputs/kernel/logs.go
View File

@ -1,9 +0,0 @@
package kernel
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (s *KernelStats) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/kernel_vmstat/logs.go
View File

@ -1,9 +0,0 @@
package kernel_vmstat
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (s *KernelVmstat) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/linux_sysctl_fs/logs.go
View File

@ -1,9 +0,0 @@
package linux_sysctl_fs
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (c *SysctlFS) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

13
inputs/logs/logs.go
View File

@ -1,13 +0,0 @@
package logs
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
type Logs struct {
Log []*logsconfig.LogsConfig `toml:"logs"`
}
func (l Logs) LogsConfig() []*logsconfig.LogsConfig {
return l.Log
}

9
inputs/mem/logs.go
View File

@ -1,9 +0,0 @@
package mem
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (c *MemStats) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

2
inputs/mysql/mysql.go
View File

@ -11,7 +11,6 @@ import (
"flashcat.cloud/categraf/config"
"flashcat.cloud/categraf/inputs"
"flashcat.cloud/categraf/inputs/logs"
"flashcat.cloud/categraf/pkg/tls"
"flashcat.cloud/categraf/types"
"github.com/go-sql-driver/mysql"
@ -155,7 +154,6 @@ func (ins *Instance) InitValidMetrics() {
type MySQL struct {
config.Interval
Instances []*Instance `toml:"instances"`
logs.Logs
Counter uint64
wg sync.WaitGroup

9
inputs/net/logs.go
View File

@ -1,9 +0,0 @@
package net
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (n *NetIOStats) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/net_response/logs.go
View File

@ -1,9 +0,0 @@
package net_response
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (n *NetResponse) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/netstat/logs.go
View File

@ -1,9 +0,0 @@
package netstat
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (n *NetStats) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/ntp/logs.go
View File

@ -1,9 +0,0 @@
package ntp
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (n *NTPStat) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/nvidia_smi/logs.go
View File

@ -1,9 +0,0 @@
package nvidia_smi
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (g *GPUStats) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

6
inputs/oracle/oracle.go
View File

@ -14,7 +14,6 @@ import (
"flashcat.cloud/categraf/config"
"flashcat.cloud/categraf/inputs"
"flashcat.cloud/categraf/inputs/logs"
"flashcat.cloud/categraf/pkg/conv"
"flashcat.cloud/categraf/types"
"github.com/godror/godror"
@ -49,9 +48,8 @@ type MetricConfig struct {
type Oracle struct {
config.Interval
Instances []OrclInstance `toml:"instances"`
Metrics []MetricConfig `toml:"metrics"`
logs.Logs
Instances []OrclInstance `toml:"instances"`
Metrics []MetricConfig `toml:"metrics"`
dbconnpool map[string]*sqlx.DB // key: instance
Counter uint64
wg sync.WaitGroup

9
inputs/ping/logs.go
View File

@ -1,9 +0,0 @@
package ping
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (c *Ping) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/processes/logs.go
View File

@ -1,9 +0,0 @@
package processes
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (c *Processes) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

9
inputs/procstat/logs.go
View File

@ -1,9 +0,0 @@
package procstat
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (c *Procstat) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

2
inputs/prometheus/prometheus.go
View File

@ -14,7 +14,6 @@ import (
"flashcat.cloud/categraf/config"
"flashcat.cloud/categraf/inputs"
"flashcat.cloud/categraf/inputs/logs"
"flashcat.cloud/categraf/parser/prometheus"
"flashcat.cloud/categraf/pkg/filter"
"flashcat.cloud/categraf/pkg/tls"
@ -100,7 +99,6 @@ func (ins *Instance) createHTTPClient() (*http.Client, error) {
type Prometheus struct {
config.Interval
Instances []*Instance `toml:"instances"`
logs.Logs
Counter uint64
wg sync.WaitGroup

2
inputs/rabbitmq/rabbitmq.go
View File

@ -13,7 +13,6 @@ import (
"flashcat.cloud/categraf/config"
"flashcat.cloud/categraf/inputs"
"flashcat.cloud/categraf/inputs/logs"
"flashcat.cloud/categraf/pkg/filter"
"flashcat.cloud/categraf/pkg/tls"
"flashcat.cloud/categraf/types"
@ -27,7 +26,6 @@ type RabbitMQ struct {
counter uint64
waitgrp sync.WaitGroup
Instances []*Instance `toml:"instances"`
logs.Logs
}
func init() {

2
inputs/redis/redis.go
View File

@ -14,7 +14,6 @@ import (
"flashcat.cloud/categraf/config"
"flashcat.cloud/categraf/inputs"
"flashcat.cloud/categraf/inputs/logs"
"flashcat.cloud/categraf/pkg/conv"
"flashcat.cloud/categraf/pkg/tls"
"flashcat.cloud/categraf/types"
@ -67,7 +66,6 @@ func (ins *Instance) Init() error {
type Redis struct {
config.Interval
Instances []*Instance `toml:"instances"`
logs.Logs
Counter uint64
wg sync.WaitGroup

9
inputs/system/logs.go
View File

@ -1,9 +0,0 @@
package system
import (
logsconfig "flashcat.cloud/categraf/config/logs"
)
func (s *SystemStats) LogsConfig() []*logsconfig.LogsConfig {
return nil
}

2
inputs/tomcat/tomcat.go
View File

@ -13,7 +13,6 @@ import (
"flashcat.cloud/categraf/config"
"flashcat.cloud/categraf/inputs"
"flashcat.cloud/categraf/inputs/logs"
"flashcat.cloud/categraf/pkg/tls"
"flashcat.cloud/categraf/types"
"github.com/toolkits/pkg/container/list"
@ -136,7 +135,6 @@ func (ins *Instance) createHTTPClient() (*http.Client, error) {
type Tomcat struct {
config.Interval
Instances []*Instance `toml:"instances"`
logs.Logs
Counter uint64
wg sync.WaitGroup

5
main.go
View File

@ -61,7 +61,10 @@ func main() {
if err := config.InitConfig(*configDir, *debugMode, *testMode); err != nil {
log.Fatalln("F! failed to init config:", err)
}
// init log config
if err := config.InitLogConfig(*configDir); err != nil {
log.Fatalln("F! failed to init config:", err)
}
// init writers
if err := writer.Init(config.Config.Writers); err != nil {
log.Fatalln("F! failed to init writer:", err)

23
pkg/cfg/cfg.go
View File

@ -39,3 +39,26 @@ func LoadConfigs(configDir string, configPtr interface{}) error {
return m.Load(configPtr)
}
func LoadConfig(configFile string, configPtr interface{}) error {
var (
loader multiconfig.Loader
)
if strings.HasSuffix(configFile, "toml") {
loader = &multiconfig.TOMLLoader{Path: configFile}
}
if strings.HasSuffix(configFile, "json") {
loader = &multiconfig.JSONLoader{Path: configFile}
}
if strings.HasSuffix(configFile, "yaml") || strings.HasSuffix(configFile, "yml") {
loader = &multiconfig.YAMLLoader{Path: configFile}
}
m := multiconfig.DefaultLoader{
Loader: multiconfig.MultiLoader(loader),
Validator: multiconfig.MultiValidator(&multiconfig.RequiredValidator{}),
}
return m.Load(configPtr)
}