Add http_middleware to transfer (#402)

* validate ui query, add aggrFun support for resample

* add http_middleware to transfer

src/modules/transfer/config/config.go

@@ -43,8 +43,9 @@ type LoggerSection struct {
 }
 
 type HTTPSection struct {
-	Enabled bool   `yaml:"enabled"`
-	Access  string `yaml:"access"`
+	Mode         string `yaml:"mode"`
+	CookieName   string `yaml:"cookieName"`
+	CookieDomain string `yaml:"cookieDomain"`
 }
 
 type RPCSection struct {

src/modules/transfer/http/health_router.go

@@ -1,4 +1,4 @@
-package routes
+package http
 
 import (
 	"fmt"

src/modules/transfer/http/http_middleware.go

@@ -0,0 +1,124 @@
+package http
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/logger"
+	"github.com/toolkits/pkg/slice"
+
+	"github.com/didi/nightingale/src/common/address"
+	"github.com/didi/nightingale/src/models"
+	"github.com/didi/nightingale/src/modules/rdb/config"
+)
+
+func shouldBeLogin() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Set("username", mustUsername(c))
+		c.Next()
+	}
+}
+
+func shouldBeRoot() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		username := mustUsername(c)
+
+		user, err := models.UserGet("username=?", username)
+		dangerous(err)
+
+		if user.IsRoot != 1 {
+			bomb("forbidden")
+		}
+
+		c.Set("username", username)
+		c.Set("user", user)
+		c.Next()
+	}
+}
+
+func shouldBeService() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		remoteAddr := c.Request.RemoteAddr
+		idx := strings.LastIndex(remoteAddr, ":")
+		ip := ""
+		if idx > 0 {
+			ip = remoteAddr[0:idx]
+		}
+
+		if ip == "127.0.0.1" {
+			c.Next()
+			return
+		}
+
+		if ip != "" && slice.ContainsString(address.GetAddresses("rdb"), ip) {
+			c.Next()
+			return
+		}
+
+		token := c.GetHeader("X-Srv-Token")
+		if token == "" {
+			c.AbortWithError(http.StatusForbidden, fmt.Errorf("X-Srv-Token is blank"))
+			return
+		}
+
+		if !slice.ContainsString(config.Config.Tokens, token) {
+			c.AbortWithError(http.StatusForbidden, fmt.Errorf("X-Srv-Token[%s] invalid", token))
+			return
+		}
+
+		c.Next()
+	}
+}
+
+func mustUsername(c *gin.Context) string {
+	username := cookieUsername(c)
+	if username == "" {
+		username = headerUsername(c)
+	}
+
+	if username == "" {
+		bomb("unauthorized")
+	}
+
+	return username
+}
+
+func cookieUsername(c *gin.Context) string {
+	return models.UsernameByUUID(readCookieUser(c))
+}
+
+func headerUsername(c *gin.Context) string {
+	token := c.GetHeader("X-User-Token")
+	if token == "" {
+		return ""
+	}
+
+	ut, err := models.UserTokenGet("token=?", token)
+	if err != nil {
+		logger.Warningf("UserTokenGet[%s] fail: %v", token, err)
+		return ""
+	}
+
+	if ut == nil {
+		return ""
+	}
+
+	return ut.Username
+}
+
+// ------------
+
+func readCookieUser(c *gin.Context) string {
+	uuid, err := c.Cookie(config.Config.HTTP.CookieName)
+	if err != nil {
+		return ""
+	}
+
+	return uuid
+}
+
+func writeCookieUser(c *gin.Context, uuid string) {
+	c.SetCookie(config.Config.HTTP.CookieName, uuid, 3600*24, "/", config.Config.HTTP.CookieDomain, false, true)
+}

src/modules/transfer/http/http_server.go

@@ -0,0 +1,70 @@
+package http
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+
+	"github.com/didi/nightingale/src/common/address"
+	"github.com/didi/nightingale/src/common/middleware"
+	"github.com/didi/nightingale/src/modules/transfer/config"
+)
+
+var srv = &http.Server{
+	ReadTimeout:    10 * time.Second,
+	WriteTimeout:   10 * time.Second,
+	MaxHeaderBytes: 1 << 20,
+}
+
+var skipPaths = []string{"/api/rdb/auth/login"}
+
+func Start() {
+	c := config.Config
+
+	loggerMid := middleware.LoggerWithConfig(middleware.LoggerConfig{SkipPaths: skipPaths})
+	recoveryMid := middleware.Recovery()
+
+	if strings.ToLower(c.HTTP.Mode) == "release" {
+		gin.SetMode(gin.ReleaseMode)
+		middleware.DisableConsoleColor()
+	}
+
+	r := gin.New()
+	r.Use(loggerMid, recoveryMid)
+
+	Config(r)
+
+	srv.Addr = address.GetHTTPListen("transfer")
+	srv.Handler = r
+
+	go func() {
+		fmt.Println("http.listening:", srv.Addr)
+		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			fmt.Printf("listening %s occur error: %s\n", srv.Addr, err)
+			os.Exit(3)
+		}
+	}()
+}
+
+// Shutdown http server
+func Shutdown() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := srv.Shutdown(ctx); err != nil {
+		fmt.Println("cannot shutdown http server:", err)
+		os.Exit(2)
+	}
+
+	// catching ctx.Done(). timeout of 5 seconds.
+	select {
+	case <-ctx.Done():
+		fmt.Println("shutdown http server timeout of 5 seconds.")
+	default:
+		fmt.Println("http server stopped")
+	}
+}

src/modules/transfer/http/push_router.go

@@ -1,4 +1,4 @@
-package routes
+package http
 
 import (
 	"github.com/didi/nightingale/src/common/dataobj"

src/modules/transfer/http/query_router.go

@@ -0,0 +1,158 @@
+package http
+
+import (
+	"github.com/didi/nightingale/src/common/dataobj"
+	"github.com/didi/nightingale/src/modules/transfer/backend"
+	"github.com/didi/nightingale/src/toolkits/http/render"
+	"github.com/didi/nightingale/src/toolkits/stats"
+
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/errors"
+	"github.com/toolkits/pkg/logger"
+)
+
+func QueryData(c *gin.Context) {
+	stats.Counter.Set("data.api.qp10s", 1)
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	var input []dataobj.QueryData
+	errors.Dangerous(c.ShouldBindJSON(&input))
+	resp := dataSource.QueryData(input)
+	render.Data(c, resp, nil)
+}
+
+func QueryDataForUI(c *gin.Context) {
+	stats.Counter.Set("data.ui.qp10s", 1)
+	var input dataobj.QueryDataForUI
+	var respData []*dataobj.QueryDataForUIResp
+
+	dangerous(c.ShouldBindJSON(&input))
+	start := input.Start
+	end := input.End
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+	resp := dataSource.QueryDataForUI(input)
+	for _, d := range resp {
+		data := &dataobj.QueryDataForUIResp{
+			Start:    d.Start,
+			End:      d.End,
+			Endpoint: d.Endpoint,
+			Nid:      d.Nid,
+			Counter:  d.Counter,
+			DsType:   d.DsType,
+			Step:     d.Step,
+			Values:   d.Values,
+		}
+		respData = append(respData, data)
+	}
+
+	if len(input.Comparisons) > 1 {
+		for i := 1; i < len(input.Comparisons); i++ {
+			comparison := input.Comparisons[i]
+			input.Start = start - comparison
+			input.End = end - comparison
+			res := dataSource.QueryDataForUI(input)
+			for _, d := range res {
+				for j := range d.Values {
+					d.Values[j].Timestamp += comparison
+				}
+
+				data := &dataobj.QueryDataForUIResp{
+					Start:      d.Start,
+					End:        d.End,
+					Endpoint:   d.Endpoint,
+					Nid:        d.Nid,
+					Counter:    d.Counter,
+					DsType:     d.DsType,
+					Step:       d.Step,
+					Values:     d.Values,
+					Comparison: comparison,
+				}
+				respData = append(respData, data)
+			}
+		}
+	}
+
+	render.Data(c, respData, nil)
+}
+
+func GetMetrics(c *gin.Context) {
+	stats.Counter.Set("metric.qp10s", 1)
+	recv := dataobj.EndpointsRecv{}
+	errors.Dangerous(c.ShouldBindJSON(&recv))
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	resp := dataSource.QueryMetrics(recv)
+
+	render.Data(c, resp, nil)
+}
+
+func GetTagPairs(c *gin.Context) {
+	stats.Counter.Set("tag.qp10s", 1)
+	recv := dataobj.EndpointMetricRecv{}
+	errors.Dangerous(c.ShouldBindJSON(&recv))
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	resp := dataSource.QueryTagPairs(recv)
+	render.Data(c, resp, nil)
+}
+
+func GetIndexByClude(c *gin.Context) {
+	stats.Counter.Set("xclude.qp10s", 1)
+	recvs := make([]dataobj.CludeRecv, 0)
+	errors.Dangerous(c.ShouldBindJSON(&recvs))
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	resp := dataSource.QueryIndexByClude(recvs)
+	render.Data(c, resp, nil)
+}
+
+func GetIndexByFullTags(c *gin.Context) {
+	stats.Counter.Set("counter.qp10s", 1)
+	recvs := make([]dataobj.IndexByFullTagsRecv, 0)
+	errors.Dangerous(c.ShouldBindJSON(&recvs))
+
+	dataSource, err := backend.GetDataSourceFor("")
+	if err != nil {
+		logger.Warningf("could not find datasource")
+		render.Message(c, err)
+		return
+	}
+
+	resp := dataSource.QueryIndexByFullTags(recvs)
+	render.Data(c, &listResp{List: resp, Count: len(resp)}, nil)
+}
+
+type listResp struct {
+	List  interface{} `json:"list"`
+	Count int         `json:"count"`
+}

src/modules/transfer/http/router_funcs.go

@@ -0,0 +1,281 @@
+package http
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/errors"
+
+	"github.com/didi/nightingale/src/models"
+)
+
+func dangerous(v interface{}) {
+	errors.Dangerous(v)
+}
+
+func bomb(format string, a ...interface{}) {
+	errors.Bomb(format, a...)
+}
+
+func bind(c *gin.Context, ptr interface{}) {
+	dangerous(c.ShouldBindJSON(ptr))
+}
+
+func urlParamStr(c *gin.Context, field string) string {
+	val := c.Param(field)
+
+	if val == "" {
+		bomb("url param[%s] is blank", field)
+	}
+
+	return val
+}
+
+func urlParamInt64(c *gin.Context, field string) int64 {
+	strval := urlParamStr(c, field)
+	intval, err := strconv.ParseInt(strval, 10, 64)
+	if err != nil {
+		bomb("cannot convert %s to int64", strval)
+	}
+
+	return intval
+}
+
+func urlParamInt(c *gin.Context, field string) int {
+	return int(urlParamInt64(c, field))
+}
+
+func queryStr(c *gin.Context, key string, defaultVal ...string) string {
+	val := c.Query(key)
+	if val != "" {
+		return val
+	}
+
+	if len(defaultVal) == 0 {
+		bomb("query param[%s] is necessary", key)
+	}
+
+	return defaultVal[0]
+}
+
+func queryInt(c *gin.Context, key string, defaultVal ...int) int {
+	strv := c.Query(key)
+	if strv != "" {
+		intv, err := strconv.Atoi(strv)
+		if err != nil {
+			bomb("cannot convert [%s] to int", strv)
+		}
+		return intv
+	}
+
+	if len(defaultVal) == 0 {
+		bomb("query param[%s] is necessary", key)
+	}
+
+	return defaultVal[0]
+}
+
+func queryInt64(c *gin.Context, key string, defaultVal ...int64) int64 {
+	strv := c.Query(key)
+	if strv != "" {
+		intv, err := strconv.ParseInt(strv, 10, 64)
+		if err != nil {
+			bomb("cannot convert [%s] to int64", strv)
+		}
+		return intv
+	}
+
+	if len(defaultVal) == 0 {
+		bomb("query param[%s] is necessary", key)
+	}
+
+	return defaultVal[0]
+}
+
+func offset(c *gin.Context, limit int) int {
+	if limit <= 0 {
+		limit = 10
+	}
+
+	page := queryInt(c, "p", 1)
+	return (page - 1) * limit
+}
+
+func renderMessage(c *gin.Context, v interface{}) {
+	if v == nil {
+		c.JSON(200, gin.H{"err": ""})
+		return
+	}
+
+	switch t := v.(type) {
+	case string:
+		c.JSON(200, gin.H{"err": t})
+	case error:
+		c.JSON(200, gin.H{"err": t.Error()})
+	}
+}
+
+func renderData(c *gin.Context, data interface{}, err error) {
+	if err == nil {
+		c.JSON(200, gin.H{"dat": data, "err": ""})
+		return
+	}
+
+	renderMessage(c, err.Error())
+}
+
+func renderZeroPage(c *gin.Context) {
+	renderData(c, gin.H{
+		"list":  []int{},
+		"total": 0,
+	}, nil)
+}
+
+// ------------
+
+type idsForm struct {
+	Ids []int64 `json:"ids"`
+}
+
+func checkPassword(passwd string) error {
+	indNum := [4]int{0, 0, 0, 0}
+	spCode := []byte{'!', '@', '#', '$', '%', '^', '&', '*', '_', '-', '~', '.', ',', '<', '>', '/', ';', ':', '|', '?', '+', '='}
+
+	if len(passwd) < 6 {
+		return fmt.Errorf("password too short")
+	}
+
+	passwdByte := []byte(passwd)
+
+	for _, i := range passwdByte {
+
+		if i >= 'A' && i <= 'Z' {
+			indNum[0] = 1
+			continue
+		}
+
+		if i >= 'a' && i <= 'z' {
+			indNum[1] = 1
+			continue
+		}
+
+		if i >= '0' && i <= '9' {
+			indNum[2] = 1
+			continue
+		}
+
+		has := false
+		for _, s := range spCode {
+			if i == s {
+				indNum[3] = 1
+				has = true
+				break
+			}
+		}
+
+		if !has {
+			return fmt.Errorf("character: %s not supported", string(i))
+		}
+
+	}
+
+	codeCount := 0
+
+	for _, i := range indNum {
+		codeCount += i
+	}
+
+	if codeCount < 4 {
+		return fmt.Errorf("password too simple")
+	}
+
+	return nil
+}
+
+// ------------
+
+func loginUsername(c *gin.Context) string {
+	value, has := c.Get("username")
+	if !has {
+		bomb("unauthorized")
+	}
+
+	if value == nil {
+		bomb("unauthorized")
+	}
+
+	return value.(string)
+}
+
+func loginUser(c *gin.Context) *models.User {
+	username := loginUsername(c)
+
+	user, err := models.UserGet("username=?", username)
+	dangerous(err)
+
+	if user == nil {
+		bomb("unauthorized")
+	}
+
+	return user
+}
+
+func loginRoot(c *gin.Context) *models.User {
+	value, has := c.Get("user")
+	if !has {
+		bomb("unauthorized")
+	}
+
+	return value.(*models.User)
+}
+
+func User(id int64) *models.User {
+	user, err := models.UserGet("id=?", id)
+	if err != nil {
+		bomb("cannot retrieve user[%d]: %v", id, err)
+	}
+
+	if user == nil {
+		bomb("no such user[%d]", id)
+	}
+
+	return user
+}
+
+func Team(id int64) *models.Team {
+	team, err := models.TeamGet("id=?", id)
+	if err != nil {
+		bomb("cannot retrieve team[%d]: %v", id, err)
+	}
+
+	if team == nil {
+		bomb("no such team[%d]", id)
+	}
+
+	return team
+}
+
+func Role(id int64) *models.Role {
+	role, err := models.RoleGet("id=?", id)
+	if err != nil {
+		bomb("cannot retrieve role[%d]: %v", id, err)
+	}
+
+	if role == nil {
+		bomb("no such role[%d]", id)
+	}
+
+	return role
+}
+
+func Node(id int64) *models.Node {
+	node, err := models.NodeGet("id=?", id)
+	dangerous(err)
+
+	if node == nil {
+		bomb("no such node[id:%d]", id)
+	}
+
+	return node
+}

src/modules/transfer/http/routes.go

@@ -1,4 +1,4 @@
-package routes
+package http
 
 import (
 	"github.com/gin-contrib/pprof"

src/modules/transfer/transfer.go

@@ -13,12 +13,10 @@ import (
 	"github.com/didi/nightingale/src/modules/transfer/backend"
 	"github.com/didi/nightingale/src/modules/transfer/config"
 	"github.com/didi/nightingale/src/modules/transfer/cron"
-	"github.com/didi/nightingale/src/modules/transfer/http/routes"
+	"github.com/didi/nightingale/src/modules/transfer/http"
 	"github.com/didi/nightingale/src/modules/transfer/rpc"
-	"github.com/didi/nightingale/src/toolkits/http"
 	"github.com/didi/nightingale/src/toolkits/stats"
 
-	"github.com/gin-gonic/gin"
 	"github.com/toolkits/pkg/file"
 	"github.com/toolkits/pkg/logger"
 	"github.com/toolkits/pkg/runner"
@@ -66,9 +64,10 @@ func main() {
 	go report.Init(cfg.Report, "rdb")
 	go rpc.Start()
 
-	r := gin.New()
-	routes.Config(r)
-	go http.Start(r, "transfer", cfg.Logger.Level)
+	// r := gin.New()
+	// routes.Config(r)
+	// go http.Start(r, "transfer", cfg.Logger.Level)
+	http.Start()
 
 	cleanup()
 }