innov
/
nightingale
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

support anonymous struct field for monapi.plugins.template (#547) 

* move get collectrule api from /api/mon to /v1/mon

* support anonymous struct field for monapi.plugins.template

* add tls with mysql, redis and mongodb

* add rdb.user.pwdExpiresAt
This commit is contained in:
yubo 2021-01-25 20:43:15 +08:00 committed by GitHub
parent 7bfd60be86
commit 8fe3457e0a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 182 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
etc/prober.yml
View File

@ -7,8 +7,7 @@ logger:
pluginsConfig: etc/plugins
report:
enabled: true
region: default
interval: 4000
timeout: 3000
api: api/hbs/heartbeat
collectRule:
token: monapi-internal-third-module-pass-fjsdi

1
src/models/user.go
View File

@ -63,6 +63,7 @@ type User struct {
LockedAt int64 `json:"locked_at" description:"locked time"`
UpdatedAt int64 `json:"updated_at" description:"user info change time"`
PwdUpdatedAt int64 `json:"pwd_updated_at" description:"password change time"`
PwdExpiresAt int64 `xorm:"-" json:"pwd_expires_at" description:"password expires time"`
LoggedAt int64 `json:"logged_at" description:"last logged time"`
CreateAt time.Time `json:"create_at" xorm:"<-"`
}

119
src/modules/monapi/collector/template.go
View File

@ -14,12 +14,12 @@ import (
var fieldCache sync.Map // map[reflect.Type]structFields
type Field struct {
skip bool `json:"-"`
Name string `json:"name,omitempty"`
Label string `json:"label,omitempty"`
Default interface{} `json:"default,omitempty"`
Enum []interface{} `json:"enum,omitempty"`
Example string `json:"example,omitempty"`
Format string `json:"format,omitempty"`
Description string `json:"description,omitempty"`
Required bool `json:"required,omitempty"`
Items *Field `json:"items,omitempty" description:"arrays's items"`
@ -27,6 +27,11 @@ type Field struct {
Ref string `json:"$ref,omitempty" description:"name of the struct ref"`
Fields []Field `json:"fields,omitempty" description:"fields of struct type"`
Definitions map[string][]Field `json:"definitions,omitempty"`
// list []Field
skip bool `json:"-"`
index []int
typ reflect.Type
}
func (p Field) String() string {
@ -44,40 +49,93 @@ func cachedTypeContent(t reflect.Type) Field {
func typeContent(t reflect.Type) Field {
definitions := map[string][]Field{t.String(): nil}
ret := Field{}
current := []Field{}
next := []Field{{typ: t}}
for i := 0; i < t.NumField(); i++ {
sf := t.Field(i)
isUnexported := sf.PkgPath != ""
if sf.Anonymous {
panic("unsupported anonymous field")
} else if isUnexported {
// Ignore unexported non-embedded fields.
continue
// Count of queued names for current level and the next.
var count, nextCount map[reflect.Type]int
// Types already visited at an earlier level.
visited := map[reflect.Type]bool{}
// Fields found.
var fields []Field
for len(next) > 0 {
current, next = next, current[:0]
count, nextCount = nextCount, map[reflect.Type]int{}
for _, f := range current {
if visited[f.typ] {
continue
}
visited[f.typ] = true
// Scan f.typ for fields to include.
for i := 0; i < f.typ.NumField(); i++ {
sf := f.typ.Field(i)
isUnexported := sf.PkgPath != ""
if sf.Anonymous {
t := sf.Type
if t.Kind() == reflect.Ptr {
t = t.Elem()
}
if isUnexported && t.Kind() != reflect.Struct {
// Ignore embedded fields of unexported non-struct types.
continue
}
// Do not ignore embedded fields of unexported struct types
// since they may have exported fields.
} else if isUnexported {
// Ignore unexported non-embedded fields.
continue
}
field := getTagOpt(sf)
if field.skip {
continue
}
index := make([]int, len(f.index)+1)
copy(index, f.index)
index[len(f.index)] = i
ft := sf.Type
if ft.Name() == "" && ft.Kind() == reflect.Ptr {
// Follow pointer.
ft = ft.Elem()
}
fieldType(ft, &field, definitions)
// Record found field and index sequence.
if field.Name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct {
field.index = index
field.typ = ft
fields = append(fields, field)
if count[f.typ] > 1 {
// If there were multiple instances, add a second,
// so that the annihilation code will see a duplicate.
// It only cares about the distinction between 1 or 2,
// so don't bother generating any more copies.
fields = append(fields, fields[len(fields)-1])
}
continue
}
// Record new anonymous struct to explore in next round.
nextCount[ft]++
if nextCount[ft] == 1 {
next = append(next, Field{index: index, typ: ft})
}
}
}
field := getTagOpt(sf)
if field.skip {
continue
}
ft := sf.Type
fieldType(ft, &field, definitions)
// Record found field and index sequence.
if field.Name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct {
ret.Fields = append(ret.Fields, field)
continue
}
panic("unsupported anonymous, struct field")
}
definitions[t.String()] = ret.Fields
definitions[t.String()] = fields
ret.Definitions = definitions
return ret
return Field{Fields: fields, Definitions: definitions}
}
// tagOptions is the string following a comma in a struct field's "json"
@ -134,6 +192,7 @@ func getTagOpt(sf reflect.StructField) (opt Field) {
opt.Name = name
opt.Label = _s(sf.Tag.Get("label"))
opt.Example = sf.Tag.Get("example")
opt.Format = sf.Tag.Get("format")
opt.Description = _s(sf.Tag.Get("description"))
if s := sf.Tag.Get("enum"); s != "" {
if err := json.Unmarshal([]byte(s), &opt.Enum); err != nil {

15
src/modules/monapi/http/router.go
View File

@ -121,8 +121,7 @@ func Config(r *gin.Engine) {
collectRulesAnonymous := r.Group("/api/mon/collect-rules")
{
collectRulesAnonymous.GET("/endpoints/:endpoint/remote", collectRulesGetByRemoteEndpoint) // for prober
collectRulesAnonymous.GET("/endpoints/:endpoint/local", collectRulesGetByLocalEndpoint) // for agent
collectRulesAnonymous.GET("/endpoints/:endpoint/local", collectRulesGetByLocalEndpoint) // for agent
}
stra := r.Group("/api/mon/stra").Use(GetCookieUser())
@ -183,14 +182,10 @@ func Config(r *gin.Engine) {
indexProxy.POST("/counter/detail", indexReq)
}
/*
v1 := r.Group("/v1/mon")
{
v1.POST("/report-detector-heartbeat", detectorHeartBeat)
v1.GET("/detectors", detectorInstanceGets)
v1.GET("/rules", collectRulesGet)
}
*/
v1 := r.Group("/v1/mon")
{
v1.GET("/collect-rules/endpoints/:endpoint/remote", collectRulesGetByRemoteEndpoint) // for prober
}
if config.Get().Logger.Level == "DEBUG" {
pprof.Register(r, "/api/monapi/debug/pprof")

3
src/modules/monapi/plugins/mongodb/mongodb.go
View File

@ -50,7 +50,7 @@ type MongodbRule struct {
GatherPerdbStats bool `label:"Per DB stats" json:"gather_perdb_stats" description:"When true, collect per database stats" default:"false"`
GatherColStats bool `label:"Col stats" json:"gather_col_stats" description:"When true, collect per collection stats" default:"false"`
ColStatsDbs []string `label:"Col stats dbs" json:"col_stats_dbs" description:"List of db where collections stats are collected, If empty, all db are concerned" example:"local" default:"[\"local\"]"`
// tlsint.ClientConfig
plugins.ClientConfig
// Ssl Ssl
}
@ -74,5 +74,6 @@ func (p *MongodbRule) TelegrafInput() (telegraf.Input, error) {
GatherColStats: p.GatherColStats,
ColStatsDbs: p.ColStatsDbs,
Log: plugins.GetLogger(),
ClientConfig: p.ClientConfig.TlsClientConfig(),
}, nil
}

2
src/modules/monapi/plugins/mysql/mysql.go
View File

@ -96,6 +96,7 @@ type MysqlRule struct {
GatherGlobalVars bool `label:"Global Vars" json:"gather_global_variables" description:"gather metrics from PERFORMANCE_SCHEMA.GLOBAL_VARIABLES" default:"true"`
IntervalSlow string `label:"Interval Slow" json:"interval_slow" description:"Some queries we may want to run less often (such as SHOW GLOBAL VARIABLES)" example:"30m"`
MetricVersion int `label:"-" json:"-"`
plugins.ClientConfig
}
func (p *MysqlRule) Validate() error {
@ -142,5 +143,6 @@ func (p *MysqlRule) TelegrafInput() (telegraf.Input, error) {
IntervalSlow: p.IntervalSlow,
MetricVersion: 2,
Log: plugins.GetLogger(),
ClientConfig: p.ClientConfig.TlsClientConfig(),
}, nil
}

12
src/modules/monapi/plugins/redis/redis.go
View File

@ -54,7 +54,8 @@ type RedisCommand struct {
type RedisRule struct {
Servers []string `label:"Servers" json:"servers,required" description:"specify servers" example:"tcp://localhost:6379"`
Commands []*RedisCommand `label:"Commands" json:"commands" description:"Optional. Specify redis commands to retrieve values"`
Password string `label:"Password" json:"password" description:"specify server password"`
Password string `label:"Password" json:"password" format:"password" description:"specify server password"`
plugins.ClientConfig
}
func (p *RedisRule) Validate() error {
@ -107,9 +108,10 @@ func (p *RedisRule) TelegrafInput() (telegraf.Input, error) {
}
return &redis.Redis{
Servers: p.Servers,
Commands: commands,
Password: p.Password,
Log: plugins.GetLogger(),
Servers: p.Servers,
Commands: commands,
Password: p.Password,
Log: plugins.GetLogger(),
ClientConfig: p.ClientConfig.TlsClientConfig(),
}, nil
}

37
src/modules/monapi/plugins/types.go Normal file
View File

@ -0,0 +1,37 @@
package plugins
import (
"github.com/didi/nightingale/src/toolkits/i18n"
"github.com/influxdata/telegraf/plugins/common/tls"
)
func init() {
i18n.DictRegister(langDict)
}
var (
langDict = map[string]map[string]string{
"zh": map[string]string{
"disables SSL certificate verification": "禁用SSL证书验证",
"verify certificates of TLS enabled servers using this CA bundle": "使用此CA文件验证服务器的证书",
"identify TLS client using this SSL certificate file": "使用此SSL证书文件标识TLS客户端",
"identify TLS client using this SSL key file": "使用此SSL密钥文件标识TLS客户端",
},
}
)
type ClientConfig struct {
InsecureSkipVerify bool `label:"Insecure Skip" json:"insecure_skip_verify" default:"false" description:"disables SSL certificate verification"`
TLSCA string `label:"CA" json:"tls_ca" format:"file" description:"verify certificates of TLS enabled servers using this CA bundle"`
TLSCert string `label:"Cert" json:"tls_cert" format:"file" description:"identify TLS client using this SSL certificate file"`
TLSKey string `label:"Key" json:"tls_key" format:"file" description:"identify TLS client using this SSL key file"`
}
func (config ClientConfig) TlsClientConfig() tls.ClientConfig {
return tls.ClientConfig{
InsecureSkipVerify: config.InsecureSkipVerify,
TLSCA: config.TLSCA,
TLSCert: config.TLSCert,
TLSKey: config.TLSKey,
}
}

2
src/modules/monapi/scache/collectrule.go
View File

@ -172,7 +172,7 @@ func (p *collectRuleCache) syncPlacement() error {
if _, exists := nodesMap[d.Region]; !exists {
nodesMap[d.Region] = make(map[string]struct{})
}
nodesMap[d.Region][d.Identity+":"+d.RPCPort] = struct{}{}
nodesMap[d.Region][d.Identity+":"+d.HTTPPort] = struct{}{}
}
}

12
src/modules/prober/cache/collectrule.go vendored
View File

@ -24,6 +24,7 @@ type CollectRuleCache struct {
TS map[int64]int64
C chan time.Time
timeout time.Duration
token string
}
func NewCollectRuleCache(cf *config.CollectRuleSection) *CollectRuleCache {
@ -31,8 +32,9 @@ func NewCollectRuleCache(cf *config.CollectRuleSection) *CollectRuleCache {
CollectRuleSection: cf,
Data: make(map[int64]*models.CollectRule),
TS: make(map[int64]int64),
timeout: time.Duration(cf.Timeout) * time.Millisecond,
C: make(chan time.Time, 1),
timeout: time.Duration(cf.Timeout) * time.Millisecond,
token: cf.Token,
}
}
@ -119,10 +121,10 @@ func (p *CollectRuleCache) syncCollectRule() error {
return fmt.Errorf("getIdent err %s", err)
}
url := fmt.Sprintf("http://%s/api/mon/collect-rules/endpoints/%s:%s/remote",
addrs[perm[i]], ident, report.Config.RPCPort)
err = httplib.Get(url).SetTimeout(p.timeout).ToJSON(&resp)
if err != nil {
url := fmt.Sprintf("http://%s/v1/mon/collect-rules/endpoints/%s:%s/remote",
addrs[perm[i]], ident, report.Config.HTTPPort)
if err = httplib.Get(url).SetTimeout(p.timeout).
Header("X-Srv-Token", p.token).ToJSON(&resp); err != nil {
logger.Warningf("get %s collect rule from remote failed, error:%v", url, err)
stats.Counter.Set("collectrule.get.err", 1)
continue

4
src/modules/rdb/auth/auth.go
View File

@ -44,3 +44,7 @@ func DeleteToken(accessToken string) error {
func Start() error {
return defaultAuth.Start()
}
func PrepareUser(user *models.User) {
defaultAuth.PrepareUser(user)
}

19
src/modules/rdb/auth/authenticator.go
View File

@ -294,6 +294,15 @@ func (p *Authenticator) Start() error {
return nil
}
func (p *Authenticator) PrepareUser(user *models.User) {
if !p.extraMode {
return
}
cf := cache.AuthConfig()
user.PwdExpiresAt = user.PwdUpdatedAt + cf.PwdExpiresIn*86400*30
}
// cleanup rdb.session & sso.token
func (p *Authenticator) cleanupSession() {
now := time.Now().Unix()
@ -432,7 +441,7 @@ func checkPassword(cf *models.AuthConfig, passwd string) error {
spCode := []byte{'!', '@', '#', '$', '%', '^', '&', '*', '_', '-', '~', '.', ',', '<', '>', '/', ';', ':', '|', '?', '+', '='}
if cf.PwdMinLenght > 0 && len(passwd) < cf.PwdMinLenght {
return _e("Password too short (min:%d) %s", cf.PwdMinLenght, cf.MustInclude())
return _e("Password too short (min:%d)", cf.PwdMinLenght)
}
passwdByte := []byte(passwd)
@ -469,19 +478,19 @@ func checkPassword(cf *models.AuthConfig, passwd string) error {
}
if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_UPPER > 0 && indNum[0] == 0 {
return _e("Invalid Password, %s", cf.MustInclude())
return _e("Invalid Password, must include %s", _s("Upper char"))
}
if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_LOWER > 0 && indNum[1] == 0 {
return _e("Invalid Password, %s", cf.MustInclude())
return _e("Invalid Password, must include %s", _s("Lower char"))
}
if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_NUMBER > 0 && indNum[2] == 0 {
return _e("Invalid Password, %s", cf.MustInclude())
return _e("Invalid Password, must include %s", _s("Number"))
}
if cf.PwdMustIncludeFlag&models.PWD_INCLUDE_SPEC_CHAR > 0 && indNum[3] == 0 {
return _e("Invalid Password, %s", cf.MustInclude())
return _e("Invalid Password, must include %s", _s("Special char"))
}
return nil

6
src/modules/rdb/config/i18n.go
View File

@ -96,8 +96,8 @@ var (
"Number": "数字",
"Special char": "特殊字符",
"Must include %s": "必须包含 %s",
"Invalid Password, %s": "密码不符合规范, %s",
"character: %s not supported": "不支持的字符 %s",
"Invalid Password, must include %s": "密码不符合规范, 必须包括 %s",
"character: %s not supported %s": "不支持的字符 %s, %s",
"Incorrect login/password %s times, you still have %s chances": "登陆失败%d次，你还有%d次机会",
"The limited sessions %d": "会话数量限制，最多%d个会话",
"Password has been expired": "密码已过期，请重置密码",
@ -106,7 +106,7 @@ var (
"User is frozen": "用户已休眠",
"User is writen off": "用户已注销",
"Minimum password length %d": "密码最小长度 %d",
"Password too short (min:%d) %s": "密码太短 (最小 %d) %s",
"Password too short (min:%d)": "密码太短 (最小 %d)",
"%s format error": "%s 所填内容不符合规范",
"%s %s format error": "%s %s 所填内容不符合规范",
"username too long (max:%d)": "用户名太长 (最长:%d)",

3
src/modules/rdb/http/router_funcs.go
View File

@ -5,6 +5,7 @@ import (
"strconv"
"github.com/didi/nightingale/src/models"
"github.com/didi/nightingale/src/modules/rdb/auth"
"github.com/didi/nightingale/src/toolkits/i18n"
"github.com/gin-gonic/gin"
"github.com/toolkits/pkg/errors"
@ -159,6 +160,8 @@ func loginUser(c *gin.Context) *models.User {
bomb("unauthorized")
}
auth.PrepareUser(user)
return user
}

4
src/modules/rdb/http/router_user.go
View File

@ -25,6 +25,7 @@ func userListGet(c *gin.Context) {
for i := 0; i < len(list); i++ {
list[i].UUID = ""
auth.PrepareUser(&list[i])
}
renderData(c, gin.H{
@ -103,6 +104,9 @@ func userAddPost(c *gin.Context) {
func userProfileGet(c *gin.Context) {
user := User(urlParamInt64(c, "id"))
user.UUID = ""
auth.PrepareUser(user)
renderData(c, user, nil)
}