add forward stat

src/server/router/router.go

@@ -108,6 +108,6 @@ func stat() gin.HandlerFunc {
 		method := c.Request.Method
 		labels := []string{"n9e-server", code, c.FullPath(), method}
 
-		promstat.RequestDuration.WithLabelValues(labels...).Observe(float64(time.Since(start).Seconds()))
+		promstat.RequestDuration.WithLabelValues(labels...).Observe(time.Since(start).Seconds())
 	}
 }

src/server/stat/stat.go

@@ -66,6 +66,15 @@ var (
 			Help:    "HTTP request latencies in seconds.",
 		}, []string{"service", "code", "path", "method"},
 	)
+
+	// 发往后端TSDB，延迟如何
+	ForwardDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Buckets: []float64{.1, 1, 10},
+			Name:    "forward_duration_seconds",
+			Help:    "Forward samples to TSDB. latencies in seconds.",
+		}, []string{"cluster", "channel_number"},
+	)
 )
 
 func Init() {
@@ -78,5 +87,6 @@ func Init() {
 		GaugeAlertQueueSize,
 		GaugeSampleQueueSize,
 		RequestDuration,
+		ForwardDuration,
 	)
 }

src/server/writer/writer.go

@@ -155,6 +155,11 @@ func (ws *WritersType) StartConsumer(index int, ch chan *prompb.TimeSeries) {
 // post post series to TSDB
 // @Author: quzhihao
 func (ws *WritersType) post(index int, series []*prompb.TimeSeries) {
+	start := time.Now()
+	defer func() {
+		promstat.ForwardDuration.WithLabelValues(config.C.ClusterName, fmt.Sprint(index)).Observe(time.Since(start).Seconds())
+	}()
+
 	header := map[string]string{"hash": fmt.Sprintf("%s-%d", config.C.Heartbeat.Endpoint, index)}
 	if len(ws.backends) == 1 {
 		for key := range ws.backends {