diff --git a/src/modules/transfer/config/config.go b/src/modules/transfer/config/config.go index 6b4b4f7c..7dad2e4e 100644 --- a/src/modules/transfer/config/config.go +++ b/src/modules/transfer/config/config.go @@ -43,8 +43,9 @@ type LoggerSection struct { } type HTTPSection struct { - Enabled bool `yaml:"enabled"` - Access string `yaml:"access"` + Mode string `yaml:"mode"` + CookieName string `yaml:"cookieName"` + CookieDomain string `yaml:"cookieDomain"` } type RPCSection struct { diff --git a/src/modules/transfer/http/routes/health_router.go b/src/modules/transfer/http/health_router.go similarity index 99% rename from src/modules/transfer/http/routes/health_router.go rename to src/modules/transfer/http/health_router.go index 89a6c620..a9de83be 100644 --- a/src/modules/transfer/http/routes/health_router.go +++ b/src/modules/transfer/http/health_router.go @@ -1,4 +1,4 @@ -package routes +package http import ( "fmt" diff --git a/src/modules/transfer/http/http_middleware.go b/src/modules/transfer/http/http_middleware.go new file mode 100644 index 00000000..5e250272 --- /dev/null +++ b/src/modules/transfer/http/http_middleware.go @@ -0,0 +1,124 @@ +package http + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/toolkits/pkg/logger" + "github.com/toolkits/pkg/slice" + + "github.com/didi/nightingale/src/common/address" + "github.com/didi/nightingale/src/models" + "github.com/didi/nightingale/src/modules/rdb/config" +) + +func shouldBeLogin() gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("username", mustUsername(c)) + c.Next() + } +} + +func shouldBeRoot() gin.HandlerFunc { + return func(c *gin.Context) { + username := mustUsername(c) + + user, err := models.UserGet("username=?", username) + dangerous(err) + + if user.IsRoot != 1 { + bomb("forbidden") + } + + c.Set("username", username) + c.Set("user", user) + c.Next() + } +} + +func shouldBeService() gin.HandlerFunc { + return func(c *gin.Context) { + remoteAddr := c.Request.RemoteAddr + idx := strings.LastIndex(remoteAddr, ":") + ip := "" + if idx > 0 { + ip = remoteAddr[0:idx] + } + + if ip == "127.0.0.1" { + c.Next() + return + } + + if ip != "" && slice.ContainsString(address.GetAddresses("rdb"), ip) { + c.Next() + return + } + + token := c.GetHeader("X-Srv-Token") + if token == "" { + c.AbortWithError(http.StatusForbidden, fmt.Errorf("X-Srv-Token is blank")) + return + } + + if !slice.ContainsString(config.Config.Tokens, token) { + c.AbortWithError(http.StatusForbidden, fmt.Errorf("X-Srv-Token[%s] invalid", token)) + return + } + + c.Next() + } +} + +func mustUsername(c *gin.Context) string { + username := cookieUsername(c) + if username == "" { + username = headerUsername(c) + } + + if username == "" { + bomb("unauthorized") + } + + return username +} + +func cookieUsername(c *gin.Context) string { + return models.UsernameByUUID(readCookieUser(c)) +} + +func headerUsername(c *gin.Context) string { + token := c.GetHeader("X-User-Token") + if token == "" { + return "" + } + + ut, err := models.UserTokenGet("token=?", token) + if err != nil { + logger.Warningf("UserTokenGet[%s] fail: %v", token, err) + return "" + } + + if ut == nil { + return "" + } + + return ut.Username +} + +// ------------ + +func readCookieUser(c *gin.Context) string { + uuid, err := c.Cookie(config.Config.HTTP.CookieName) + if err != nil { + return "" + } + + return uuid +} + +func writeCookieUser(c *gin.Context, uuid string) { + c.SetCookie(config.Config.HTTP.CookieName, uuid, 3600*24, "/", config.Config.HTTP.CookieDomain, false, true) +} diff --git a/src/modules/transfer/http/http_server.go b/src/modules/transfer/http/http_server.go new file mode 100644 index 00000000..1f9b080d --- /dev/null +++ b/src/modules/transfer/http/http_server.go @@ -0,0 +1,70 @@ +package http + +import ( + "context" + "fmt" + "net/http" + "os" + "strings" + "time" + + "github.com/gin-gonic/gin" + + "github.com/didi/nightingale/src/common/address" + "github.com/didi/nightingale/src/common/middleware" + "github.com/didi/nightingale/src/modules/transfer/config" +) + +var srv = &http.Server{ + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, +} + +var skipPaths = []string{"/api/rdb/auth/login"} + +func Start() { + c := config.Config + + loggerMid := middleware.LoggerWithConfig(middleware.LoggerConfig{SkipPaths: skipPaths}) + recoveryMid := middleware.Recovery() + + if strings.ToLower(c.HTTP.Mode) == "release" { + gin.SetMode(gin.ReleaseMode) + middleware.DisableConsoleColor() + } + + r := gin.New() + r.Use(loggerMid, recoveryMid) + + Config(r) + + srv.Addr = address.GetHTTPListen("transfer") + srv.Handler = r + + go func() { + fmt.Println("http.listening:", srv.Addr) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + fmt.Printf("listening %s occur error: %s\n", srv.Addr, err) + os.Exit(3) + } + }() +} + +// Shutdown http server +func Shutdown() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + fmt.Println("cannot shutdown http server:", err) + os.Exit(2) + } + + // catching ctx.Done(). timeout of 5 seconds. + select { + case <-ctx.Done(): + fmt.Println("shutdown http server timeout of 5 seconds.") + default: + fmt.Println("http server stopped") + } +} diff --git a/src/modules/transfer/http/routes/push_router.go b/src/modules/transfer/http/push_router.go similarity index 97% rename from src/modules/transfer/http/routes/push_router.go rename to src/modules/transfer/http/push_router.go index 1d02c22d..59503501 100644 --- a/src/modules/transfer/http/routes/push_router.go +++ b/src/modules/transfer/http/push_router.go @@ -1,4 +1,4 @@ -package routes +package http import ( "github.com/didi/nightingale/src/common/dataobj" diff --git a/src/modules/transfer/http/query_router.go b/src/modules/transfer/http/query_router.go new file mode 100644 index 00000000..31f6be95 --- /dev/null +++ b/src/modules/transfer/http/query_router.go @@ -0,0 +1,158 @@ +package http + +import ( + "github.com/didi/nightingale/src/common/dataobj" + "github.com/didi/nightingale/src/modules/transfer/backend" + "github.com/didi/nightingale/src/toolkits/http/render" + "github.com/didi/nightingale/src/toolkits/stats" + + "github.com/gin-gonic/gin" + "github.com/toolkits/pkg/errors" + "github.com/toolkits/pkg/logger" +) + +func QueryData(c *gin.Context) { + stats.Counter.Set("data.api.qp10s", 1) + + dataSource, err := backend.GetDataSourceFor("") + if err != nil { + logger.Warningf("could not find datasource") + render.Message(c, err) + return + } + + var input []dataobj.QueryData + errors.Dangerous(c.ShouldBindJSON(&input)) + resp := dataSource.QueryData(input) + render.Data(c, resp, nil) +} + +func QueryDataForUI(c *gin.Context) { + stats.Counter.Set("data.ui.qp10s", 1) + var input dataobj.QueryDataForUI + var respData []*dataobj.QueryDataForUIResp + + dangerous(c.ShouldBindJSON(&input)) + start := input.Start + end := input.End + + dataSource, err := backend.GetDataSourceFor("") + if err != nil { + logger.Warningf("could not find datasource") + render.Message(c, err) + return + } + resp := dataSource.QueryDataForUI(input) + for _, d := range resp { + data := &dataobj.QueryDataForUIResp{ + Start: d.Start, + End: d.End, + Endpoint: d.Endpoint, + Nid: d.Nid, + Counter: d.Counter, + DsType: d.DsType, + Step: d.Step, + Values: d.Values, + } + respData = append(respData, data) + } + + if len(input.Comparisons) > 1 { + for i := 1; i < len(input.Comparisons); i++ { + comparison := input.Comparisons[i] + input.Start = start - comparison + input.End = end - comparison + res := dataSource.QueryDataForUI(input) + for _, d := range res { + for j := range d.Values { + d.Values[j].Timestamp += comparison + } + + data := &dataobj.QueryDataForUIResp{ + Start: d.Start, + End: d.End, + Endpoint: d.Endpoint, + Nid: d.Nid, + Counter: d.Counter, + DsType: d.DsType, + Step: d.Step, + Values: d.Values, + Comparison: comparison, + } + respData = append(respData, data) + } + } + } + + render.Data(c, respData, nil) +} + +func GetMetrics(c *gin.Context) { + stats.Counter.Set("metric.qp10s", 1) + recv := dataobj.EndpointsRecv{} + errors.Dangerous(c.ShouldBindJSON(&recv)) + + dataSource, err := backend.GetDataSourceFor("") + if err != nil { + logger.Warningf("could not find datasource") + render.Message(c, err) + return + } + + resp := dataSource.QueryMetrics(recv) + + render.Data(c, resp, nil) +} + +func GetTagPairs(c *gin.Context) { + stats.Counter.Set("tag.qp10s", 1) + recv := dataobj.EndpointMetricRecv{} + errors.Dangerous(c.ShouldBindJSON(&recv)) + + dataSource, err := backend.GetDataSourceFor("") + if err != nil { + logger.Warningf("could not find datasource") + render.Message(c, err) + return + } + + resp := dataSource.QueryTagPairs(recv) + render.Data(c, resp, nil) +} + +func GetIndexByClude(c *gin.Context) { + stats.Counter.Set("xclude.qp10s", 1) + recvs := make([]dataobj.CludeRecv, 0) + errors.Dangerous(c.ShouldBindJSON(&recvs)) + + dataSource, err := backend.GetDataSourceFor("") + if err != nil { + logger.Warningf("could not find datasource") + render.Message(c, err) + return + } + + resp := dataSource.QueryIndexByClude(recvs) + render.Data(c, resp, nil) +} + +func GetIndexByFullTags(c *gin.Context) { + stats.Counter.Set("counter.qp10s", 1) + recvs := make([]dataobj.IndexByFullTagsRecv, 0) + errors.Dangerous(c.ShouldBindJSON(&recvs)) + + dataSource, err := backend.GetDataSourceFor("") + if err != nil { + logger.Warningf("could not find datasource") + render.Message(c, err) + return + } + + resp := dataSource.QueryIndexByFullTags(recvs) + render.Data(c, &listResp{List: resp, Count: len(resp)}, nil) +} + +type listResp struct { + List interface{} `json:"list"` + Count int `json:"count"` +} diff --git a/src/modules/transfer/http/router_funcs.go b/src/modules/transfer/http/router_funcs.go new file mode 100644 index 00000000..99c906ec --- /dev/null +++ b/src/modules/transfer/http/router_funcs.go @@ -0,0 +1,281 @@ +package http + +import ( + "fmt" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/toolkits/pkg/errors" + + "github.com/didi/nightingale/src/models" +) + +func dangerous(v interface{}) { + errors.Dangerous(v) +} + +func bomb(format string, a ...interface{}) { + errors.Bomb(format, a...) +} + +func bind(c *gin.Context, ptr interface{}) { + dangerous(c.ShouldBindJSON(ptr)) +} + +func urlParamStr(c *gin.Context, field string) string { + val := c.Param(field) + + if val == "" { + bomb("url param[%s] is blank", field) + } + + return val +} + +func urlParamInt64(c *gin.Context, field string) int64 { + strval := urlParamStr(c, field) + intval, err := strconv.ParseInt(strval, 10, 64) + if err != nil { + bomb("cannot convert %s to int64", strval) + } + + return intval +} + +func urlParamInt(c *gin.Context, field string) int { + return int(urlParamInt64(c, field)) +} + +func queryStr(c *gin.Context, key string, defaultVal ...string) string { + val := c.Query(key) + if val != "" { + return val + } + + if len(defaultVal) == 0 { + bomb("query param[%s] is necessary", key) + } + + return defaultVal[0] +} + +func queryInt(c *gin.Context, key string, defaultVal ...int) int { + strv := c.Query(key) + if strv != "" { + intv, err := strconv.Atoi(strv) + if err != nil { + bomb("cannot convert [%s] to int", strv) + } + return intv + } + + if len(defaultVal) == 0 { + bomb("query param[%s] is necessary", key) + } + + return defaultVal[0] +} + +func queryInt64(c *gin.Context, key string, defaultVal ...int64) int64 { + strv := c.Query(key) + if strv != "" { + intv, err := strconv.ParseInt(strv, 10, 64) + if err != nil { + bomb("cannot convert [%s] to int64", strv) + } + return intv + } + + if len(defaultVal) == 0 { + bomb("query param[%s] is necessary", key) + } + + return defaultVal[0] +} + +func offset(c *gin.Context, limit int) int { + if limit <= 0 { + limit = 10 + } + + page := queryInt(c, "p", 1) + return (page - 1) * limit +} + +func renderMessage(c *gin.Context, v interface{}) { + if v == nil { + c.JSON(200, gin.H{"err": ""}) + return + } + + switch t := v.(type) { + case string: + c.JSON(200, gin.H{"err": t}) + case error: + c.JSON(200, gin.H{"err": t.Error()}) + } +} + +func renderData(c *gin.Context, data interface{}, err error) { + if err == nil { + c.JSON(200, gin.H{"dat": data, "err": ""}) + return + } + + renderMessage(c, err.Error()) +} + +func renderZeroPage(c *gin.Context) { + renderData(c, gin.H{ + "list": []int{}, + "total": 0, + }, nil) +} + +// ------------ + +type idsForm struct { + Ids []int64 `json:"ids"` +} + +func checkPassword(passwd string) error { + indNum := [4]int{0, 0, 0, 0} + spCode := []byte{'!', '@', '#', '$', '%', '^', '&', '*', '_', '-', '~', '.', ',', '<', '>', '/', ';', ':', '|', '?', '+', '='} + + if len(passwd) < 6 { + return fmt.Errorf("password too short") + } + + passwdByte := []byte(passwd) + + for _, i := range passwdByte { + + if i >= 'A' && i <= 'Z' { + indNum[0] = 1 + continue + } + + if i >= 'a' && i <= 'z' { + indNum[1] = 1 + continue + } + + if i >= '0' && i <= '9' { + indNum[2] = 1 + continue + } + + has := false + for _, s := range spCode { + if i == s { + indNum[3] = 1 + has = true + break + } + } + + if !has { + return fmt.Errorf("character: %s not supported", string(i)) + } + + } + + codeCount := 0 + + for _, i := range indNum { + codeCount += i + } + + if codeCount < 4 { + return fmt.Errorf("password too simple") + } + + return nil +} + +// ------------ + +func loginUsername(c *gin.Context) string { + value, has := c.Get("username") + if !has { + bomb("unauthorized") + } + + if value == nil { + bomb("unauthorized") + } + + return value.(string) +} + +func loginUser(c *gin.Context) *models.User { + username := loginUsername(c) + + user, err := models.UserGet("username=?", username) + dangerous(err) + + if user == nil { + bomb("unauthorized") + } + + return user +} + +func loginRoot(c *gin.Context) *models.User { + value, has := c.Get("user") + if !has { + bomb("unauthorized") + } + + return value.(*models.User) +} + +func User(id int64) *models.User { + user, err := models.UserGet("id=?", id) + if err != nil { + bomb("cannot retrieve user[%d]: %v", id, err) + } + + if user == nil { + bomb("no such user[%d]", id) + } + + return user +} + +func Team(id int64) *models.Team { + team, err := models.TeamGet("id=?", id) + if err != nil { + bomb("cannot retrieve team[%d]: %v", id, err) + } + + if team == nil { + bomb("no such team[%d]", id) + } + + return team +} + +func Role(id int64) *models.Role { + role, err := models.RoleGet("id=?", id) + if err != nil { + bomb("cannot retrieve role[%d]: %v", id, err) + } + + if role == nil { + bomb("no such role[%d]", id) + } + + return role +} + +func Node(id int64) *models.Node { + node, err := models.NodeGet("id=?", id) + dangerous(err) + + if node == nil { + bomb("no such node[id:%d]", id) + } + + return node +} diff --git a/src/modules/transfer/http/routes/routes.go b/src/modules/transfer/http/routes.go similarity index 98% rename from src/modules/transfer/http/routes/routes.go rename to src/modules/transfer/http/routes.go index fc27f3de..f45af474 100644 --- a/src/modules/transfer/http/routes/routes.go +++ b/src/modules/transfer/http/routes.go @@ -1,4 +1,4 @@ -package routes +package http import ( "github.com/gin-contrib/pprof" diff --git a/src/modules/transfer/transfer.go b/src/modules/transfer/transfer.go index 6e842eba..3248611e 100644 --- a/src/modules/transfer/transfer.go +++ b/src/modules/transfer/transfer.go @@ -13,12 +13,10 @@ import ( "github.com/didi/nightingale/src/modules/transfer/backend" "github.com/didi/nightingale/src/modules/transfer/config" "github.com/didi/nightingale/src/modules/transfer/cron" - "github.com/didi/nightingale/src/modules/transfer/http/routes" + "github.com/didi/nightingale/src/modules/transfer/http" "github.com/didi/nightingale/src/modules/transfer/rpc" - "github.com/didi/nightingale/src/toolkits/http" "github.com/didi/nightingale/src/toolkits/stats" - "github.com/gin-gonic/gin" "github.com/toolkits/pkg/file" "github.com/toolkits/pkg/logger" "github.com/toolkits/pkg/runner" @@ -66,9 +64,10 @@ func main() { go report.Init(cfg.Report, "rdb") go rpc.Start() - r := gin.New() - routes.Config(r) - go http.Start(r, "transfer", cfg.Logger.Level) + // r := gin.New() + // routes.Config(r) + // go http.Start(r, "transfer", cfg.Logger.Level) + http.Start() cleanup() }