diff --git a/etc/prober.yml b/etc/prober.yml index 19518c0c..1decad5c 100644 --- a/etc/prober.yml +++ b/etc/prober.yml @@ -7,8 +7,7 @@ logger: pluginsConfig: etc/plugins report: - enabled: true region: default - interval: 4000 - timeout: 3000 - api: api/hbs/heartbeat + +collectRule: + token: monapi-internal-third-module-pass-fjsdi diff --git a/src/models/user.go b/src/models/user.go index 7e36a6e9..c4ef239e 100644 --- a/src/models/user.go +++ b/src/models/user.go @@ -63,6 +63,7 @@ type User struct { LockedAt int64 `json:"locked_at" description:"locked time"` UpdatedAt int64 `json:"updated_at" description:"user info change time"` PwdUpdatedAt int64 `json:"pwd_updated_at" description:"password change time"` + PwdExpiresAt int64 `xorm:"-" json:"pwd_expires_at" description:"password expires time"` LoggedAt int64 `json:"logged_at" description:"last logged time"` CreateAt time.Time `json:"create_at" xorm:"<-"` } diff --git a/src/modules/monapi/collector/template.go b/src/modules/monapi/collector/template.go index d827338c..ba12b7b5 100644 --- a/src/modules/monapi/collector/template.go +++ b/src/modules/monapi/collector/template.go @@ -14,12 +14,12 @@ import ( var fieldCache sync.Map // map[reflect.Type]structFields type Field struct { - skip bool `json:"-"` Name string `json:"name,omitempty"` Label string `json:"label,omitempty"` Default interface{} `json:"default,omitempty"` Enum []interface{} `json:"enum,omitempty"` Example string `json:"example,omitempty"` + Format string `json:"format,omitempty"` Description string `json:"description,omitempty"` Required bool `json:"required,omitempty"` Items *Field `json:"items,omitempty" description:"arrays's items"` @@ -27,6 +27,11 @@ type Field struct { Ref string `json:"$ref,omitempty" description:"name of the struct ref"` Fields []Field `json:"fields,omitempty" description:"fields of struct type"` Definitions map[string][]Field `json:"definitions,omitempty"` + + // list []Field + skip bool `json:"-"` + index []int + typ reflect.Type } func (p Field) String() string { @@ -44,40 +49,93 @@ func cachedTypeContent(t reflect.Type) Field { func typeContent(t reflect.Type) Field { definitions := map[string][]Field{t.String(): nil} - ret := Field{} + current := []Field{} + next := []Field{{typ: t}} - for i := 0; i < t.NumField(); i++ { - sf := t.Field(i) - isUnexported := sf.PkgPath != "" - if sf.Anonymous { - panic("unsupported anonymous field") - } else if isUnexported { - // Ignore unexported non-embedded fields. - continue + // Count of queued names for current level and the next. + var count, nextCount map[reflect.Type]int + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []Field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.typ] { + continue + } + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + isUnexported := sf.PkgPath != "" + if sf.Anonymous { + t := sf.Type + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if isUnexported && t.Kind() != reflect.Struct { + // Ignore embedded fields of unexported non-struct types. + continue + } + // Do not ignore embedded fields of unexported struct types + // since they may have exported fields. + } else if isUnexported { + // Ignore unexported non-embedded fields. + continue + } + + field := getTagOpt(sf) + if field.skip { + continue + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + + ft := sf.Type + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + fieldType(ft, &field, definitions) + + // Record found field and index sequence. + if field.Name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + field.index = index + field.typ = ft + + fields = append(fields, field) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + next = append(next, Field{index: index, typ: ft}) + } + + } } - - field := getTagOpt(sf) - if field.skip { - continue - } - ft := sf.Type - - fieldType(ft, &field, definitions) - - // Record found field and index sequence. - if field.Name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { - ret.Fields = append(ret.Fields, field) - continue - } - - panic("unsupported anonymous, struct field") } - definitions[t.String()] = ret.Fields + definitions[t.String()] = fields - ret.Definitions = definitions - - return ret + return Field{Fields: fields, Definitions: definitions} } // tagOptions is the string following a comma in a struct field's "json" @@ -134,6 +192,7 @@ func getTagOpt(sf reflect.StructField) (opt Field) { opt.Name = name opt.Label = _s(sf.Tag.Get("label")) opt.Example = sf.Tag.Get("example") + opt.Format = sf.Tag.Get("format") opt.Description = _s(sf.Tag.Get("description")) if s := sf.Tag.Get("enum"); s != "" { if err := json.Unmarshal([]byte(s), &opt.Enum); err != nil { diff --git a/src/modules/monapi/http/router.go b/src/modules/monapi/http/router.go index ceaf41b4..f4a7902c 100644 --- a/src/modules/monapi/http/router.go +++ b/src/modules/monapi/http/router.go @@ -121,8 +121,7 @@ func Config(r *gin.Engine) { collectRulesAnonymous := r.Group("/api/mon/collect-rules") { - collectRulesAnonymous.GET("/endpoints/:endpoint/remote", collectRulesGetByRemoteEndpoint) // for prober - collectRulesAnonymous.GET("/endpoints/:endpoint/local", collectRulesGetByLocalEndpoint) // for agent + collectRulesAnonymous.GET("/endpoints/:endpoint/local", collectRulesGetByLocalEndpoint) // for agent } stra := r.Group("/api/mon/stra").Use(GetCookieUser()) @@ -183,14 +182,10 @@ func Config(r *gin.Engine) { indexProxy.POST("/counter/detail", indexReq) } - /* - v1 := r.Group("/v1/mon") - { - v1.POST("/report-detector-heartbeat", detectorHeartBeat) - v1.GET("/detectors", detectorInstanceGets) - v1.GET("/rules", collectRulesGet) - } - */ + v1 := r.Group("/v1/mon") + { + v1.GET("/collect-rules/endpoints/:endpoint/remote", collectRulesGetByRemoteEndpoint) // for prober + } if config.Get().Logger.Level == "DEBUG" { pprof.Register(r, "/api/monapi/debug/pprof") diff --git a/src/modules/monapi/plugins/mongodb/mongodb.go b/src/modules/monapi/plugins/mongodb/mongodb.go index 0a931f1a..208ee98b 100644 --- a/src/modules/monapi/plugins/mongodb/mongodb.go +++ b/src/modules/monapi/plugins/mongodb/mongodb.go @@ -50,7 +50,7 @@ type MongodbRule struct { GatherPerdbStats bool `label:"Per DB stats" json:"gather_perdb_stats" description:"When true, collect per database stats" default:"false"` GatherColStats bool `label:"Col stats" json:"gather_col_stats" description:"When true, collect per collection stats" default:"false"` ColStatsDbs []string `label:"Col stats dbs" json:"col_stats_dbs" description:"List of db where collections stats are collected, If empty, all db are concerned" example:"local" default:"[\"local\"]"` - // tlsint.ClientConfig + plugins.ClientConfig // Ssl Ssl } @@ -74,5 +74,6 @@ func (p *MongodbRule) TelegrafInput() (telegraf.Input, error) { GatherColStats: p.GatherColStats, ColStatsDbs: p.ColStatsDbs, Log: plugins.GetLogger(), + ClientConfig: p.ClientConfig.TlsClientConfig(), }, nil } diff --git a/src/modules/monapi/plugins/mysql/mysql.go b/src/modules/monapi/plugins/mysql/mysql.go index 6110ae09..13a67305 100644 --- a/src/modules/monapi/plugins/mysql/mysql.go +++ b/src/modules/monapi/plugins/mysql/mysql.go @@ -96,6 +96,7 @@ type MysqlRule struct { GatherGlobalVars bool `label:"Global Vars" json:"gather_global_variables" description:"gather metrics from PERFORMANCE_SCHEMA.GLOBAL_VARIABLES" default:"true"` IntervalSlow string `label:"Interval Slow" json:"interval_slow" description:"Some queries we may want to run less often (such as SHOW GLOBAL VARIABLES)" example:"30m"` MetricVersion int `label:"-" json:"-"` + plugins.ClientConfig } func (p *MysqlRule) Validate() error { @@ -142,5 +143,6 @@ func (p *MysqlRule) TelegrafInput() (telegraf.Input, error) { IntervalSlow: p.IntervalSlow, MetricVersion: 2, Log: plugins.GetLogger(), + ClientConfig: p.ClientConfig.TlsClientConfig(), }, nil } diff --git a/src/modules/monapi/plugins/redis/redis.go b/src/modules/monapi/plugins/redis/redis.go index b9fc6fea..c9279aad 100644 --- a/src/modules/monapi/plugins/redis/redis.go +++ b/src/modules/monapi/plugins/redis/redis.go @@ -54,7 +54,8 @@ type RedisCommand struct { type RedisRule struct { Servers []string `label:"Servers" json:"servers,required" description:"specify servers" example:"tcp://localhost:6379"` Commands []*RedisCommand `label:"Commands" json:"commands" description:"Optional. Specify redis commands to retrieve values"` - Password string `label:"Password" json:"password" description:"specify server password"` + Password string `label:"Password" json:"password" format:"password" description:"specify server password"` + plugins.ClientConfig } func (p *RedisRule) Validate() error { @@ -107,9 +108,10 @@ func (p *RedisRule) TelegrafInput() (telegraf.Input, error) { } return &redis.Redis{ - Servers: p.Servers, - Commands: commands, - Password: p.Password, - Log: plugins.GetLogger(), + Servers: p.Servers, + Commands: commands, + Password: p.Password, + Log: plugins.GetLogger(), + ClientConfig: p.ClientConfig.TlsClientConfig(), }, nil } diff --git a/src/modules/monapi/plugins/types.go b/src/modules/monapi/plugins/types.go new file mode 100644 index 00000000..a35fdde0 --- /dev/null +++ b/src/modules/monapi/plugins/types.go @@ -0,0 +1,37 @@ +package plugins + +import ( + "github.com/didi/nightingale/src/toolkits/i18n" + "github.com/influxdata/telegraf/plugins/common/tls" +) + +func init() { + i18n.DictRegister(langDict) +} + +var ( + langDict = map[string]map[string]string{ + "zh": map[string]string{ + "disables SSL certificate verification": "禁用SSL证书验证", + "verify certificates of TLS enabled servers using this CA bundle": "使用此CA文件验证服务器的证书", + "identify TLS client using this SSL certificate file": "使用此SSL证书文件标识TLS客户端", + "identify TLS client using this SSL key file": "使用此SSL密钥文件标识TLS客户端", + }, + } +) + +type ClientConfig struct { + InsecureSkipVerify bool `label:"Insecure Skip" json:"insecure_skip_verify" default:"false" description:"disables SSL certificate verification"` + TLSCA string `label:"CA" json:"tls_ca" format:"file" description:"verify certificates of TLS enabled servers using this CA bundle"` + TLSCert string `label:"Cert" json:"tls_cert" format:"file" description:"identify TLS client using this SSL certificate file"` + TLSKey string `label:"Key" json:"tls_key" format:"file" description:"identify TLS client using this SSL key file"` +} + +func (config ClientConfig) TlsClientConfig() tls.ClientConfig { + return tls.ClientConfig{ + InsecureSkipVerify: config.InsecureSkipVerify, + TLSCA: config.TLSCA, + TLSCert: config.TLSCert, + TLSKey: config.TLSKey, + } +} diff --git a/src/modules/monapi/scache/collectrule.go b/src/modules/monapi/scache/collectrule.go index 0d1d73cf..379f9e2f 100644 --- a/src/modules/monapi/scache/collectrule.go +++ b/src/modules/monapi/scache/collectrule.go @@ -172,7 +172,7 @@ func (p *collectRuleCache) syncPlacement() error { if _, exists := nodesMap[d.Region]; !exists { nodesMap[d.Region] = make(map[string]struct{}) } - nodesMap[d.Region][d.Identity+":"+d.RPCPort] = struct{}{} + nodesMap[d.Region][d.Identity+":"+d.HTTPPort] = struct{}{} } } diff --git a/src/modules/prober/cache/collectrule.go b/src/modules/prober/cache/collectrule.go index ae2afc90..0a570880 100644 --- a/src/modules/prober/cache/collectrule.go +++ b/src/modules/prober/cache/collectrule.go @@ -24,6 +24,7 @@ type CollectRuleCache struct { TS map[int64]int64 C chan time.Time timeout time.Duration + token string } func NewCollectRuleCache(cf *config.CollectRuleSection) *CollectRuleCache { @@ -31,8 +32,9 @@ func NewCollectRuleCache(cf *config.CollectRuleSection) *CollectRuleCache { CollectRuleSection: cf, Data: make(map[int64]*models.CollectRule), TS: make(map[int64]int64), - timeout: time.Duration(cf.Timeout) * time.Millisecond, C: make(chan time.Time, 1), + timeout: time.Duration(cf.Timeout) * time.Millisecond, + token: cf.Token, } } @@ -119,10 +121,10 @@ func (p *CollectRuleCache) syncCollectRule() error { return fmt.Errorf("getIdent err %s", err) } - url := fmt.Sprintf("http://%s/api/mon/collect-rules/endpoints/%s:%s/remote", - addrs[perm[i]], ident, report.Config.RPCPort) - err = httplib.Get(url).SetTimeout(p.timeout).ToJSON(&resp) - if err != nil { + url := fmt.Sprintf("http://%s/v1/mon/collect-rules/endpoints/%s:%s/remote", + addrs[perm[i]], ident, report.Config.HTTPPort) + if err = httplib.Get(url).SetTimeout(p.timeout). + Header("X-Srv-Token", p.token).ToJSON(&resp); err != nil { logger.Warningf("get %s collect rule from remote failed, error:%v", url, err) stats.Counter.Set("collectrule.get.err", 1) continue diff --git a/src/modules/rdb/auth/auth.go b/src/modules/rdb/auth/auth.go index 099ee4d9..8b755cb5 100644 --- a/src/modules/rdb/auth/auth.go +++ b/src/modules/rdb/auth/auth.go @@ -44,3 +44,7 @@ func DeleteToken(accessToken string) error { func Start() error { return defaultAuth.Start() } + +func PrepareUser(user *models.User) { + defaultAuth.PrepareUser(user) +} diff --git a/src/modules/rdb/auth/authenticator.go b/src/modules/rdb/auth/authenticator.go index b0633868..2c56b008 100644 --- a/src/modules/rdb/auth/authenticator.go +++ b/src/modules/rdb/auth/authenticator.go @@ -294,6 +294,15 @@ func (p *Authenticator) Start() error { return nil } +func (p *Authenticator) PrepareUser(user *models.User) { + if !p.extraMode { + return + } + + cf := cache.AuthConfig() + user.PwdExpiresAt = user.PwdUpdatedAt + cf.PwdExpiresIn*86400*30 +} + // cleanup rdb.session & sso.token func (p *Authenticator) cleanupSession() { now := time.Now().Unix() @@ -432,7 +441,7 @@ func checkPassword(cf *models.AuthConfig, passwd string) error { spCode := []byte{'!', '@', '#', '$', '%', '^', '&', '*', '_', '-', '~', '.', ',', '<', '>', '/', ';', ':', '|', '?', '+', '='} if cf.PwdMinLenght > 0 && len(passwd) < cf.PwdMinLenght { - return _e("Password too short (min:%d) %s", cf.PwdMinLenght, cf.MustInclude()) + return _e("Password too short (min:%d)", cf.PwdMinLenght) } passwdByte := []byte(passwd) @@ -469,19 +478,19 @@ func checkPassword(cf *models.AuthConfig, passwd string) error { } if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_UPPER > 0 && indNum[0] == 0 { - return _e("Invalid Password, %s", cf.MustInclude()) + return _e("Invalid Password, must include %s", _s("Upper char")) } if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_LOWER > 0 && indNum[1] == 0 { - return _e("Invalid Password, %s", cf.MustInclude()) + return _e("Invalid Password, must include %s", _s("Lower char")) } if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_NUMBER > 0 && indNum[2] == 0 { - return _e("Invalid Password, %s", cf.MustInclude()) + return _e("Invalid Password, must include %s", _s("Number")) } if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_SPEC_CHAR > 0 && indNum[3] == 0 { - return _e("Invalid Password, %s", cf.MustInclude()) + return _e("Invalid Password, must include %s", _s("Special char")) } return nil diff --git a/src/modules/rdb/config/i18n.go b/src/modules/rdb/config/i18n.go index ab07498b..9f05e8c4 100644 --- a/src/modules/rdb/config/i18n.go +++ b/src/modules/rdb/config/i18n.go @@ -96,8 +96,8 @@ var ( "Number": "数字", "Special char": "特殊字符", "Must include %s": "必须包含 %s", - "Invalid Password, %s": "密码不符合规范, %s", - "character: %s not supported": "不支持的字符 %s", + "Invalid Password, must include %s": "密码不符合规范, 必须包括 %s", + "character: %s not supported %s": "不支持的字符 %s, %s", "Incorrect login/password %s times, you still have %s chances": "登陆失败%d次，你还有%d次机会", "The limited sessions %d": "会话数量限制，最多%d个会话", "Password has been expired": "密码已过期，请重置密码", @@ -106,7 +106,7 @@ var ( "User is frozen": "用户已休眠", "User is writen off": "用户已注销", "Minimum password length %d": "密码最小长度 %d", - "Password too short (min:%d) %s": "密码太短 (最小 %d) %s", + "Password too short (min:%d)": "密码太短 (最小 %d)", "%s format error": "%s 所填内容不符合规范", "%s %s format error": "%s %s 所填内容不符合规范", "username too long (max:%d)": "用户名太长 (最长:%d)", diff --git a/src/modules/rdb/http/router_funcs.go b/src/modules/rdb/http/router_funcs.go index 1b85d8a1..7f64801d 100644 --- a/src/modules/rdb/http/router_funcs.go +++ b/src/modules/rdb/http/router_funcs.go @@ -5,6 +5,7 @@ import ( "strconv" "github.com/didi/nightingale/src/models" + "github.com/didi/nightingale/src/modules/rdb/auth" "github.com/didi/nightingale/src/toolkits/i18n" "github.com/gin-gonic/gin" "github.com/toolkits/pkg/errors" @@ -159,6 +160,8 @@ func loginUser(c *gin.Context) *models.User { bomb("unauthorized") } + auth.PrepareUser(user) + return user } diff --git a/src/modules/rdb/http/router_user.go b/src/modules/rdb/http/router_user.go index 7693f043..267d032d 100644 --- a/src/modules/rdb/http/router_user.go +++ b/src/modules/rdb/http/router_user.go @@ -25,6 +25,7 @@ func userListGet(c *gin.Context) { for i := 0; i < len(list); i++ { list[i].UUID = "" + auth.PrepareUser(&list[i]) } renderData(c, gin.H{ @@ -103,6 +104,9 @@ func userAddPost(c *gin.Context) { func userProfileGet(c *gin.Context) { user := User(urlParamInt64(c, "id")) user.UUID = "" + + auth.PrepareUser(user) + renderData(c, user, nil) }