Add Promehtues metrics for Grafana close #61 (#85)

* Add Promehtues metrics for Grafana close #61

* remove debug code

* Add doc and Grafana dashboard

* PR comments fixes

* Remove macros

README.md

@@ -383,6 +383,8 @@ Right now it does not have much info but in the future we are planning to add th
 debugging and management info. If you go to `:6379/metrics` url you will see some prometheus
 compatible metrics.
 
+The Prometheus exported metrics are compatible with the Grafana dashboard [see here](examples/grafana/dashboard.json). 
+
 Important! Http console is meant to be accessed within a safe network.
 If you expose Dragonfly's TCP port externally, it is advised to disable the console
 with `--http_admin_console=false` or `--nohttp_admin_console`.

src/server/server_family.cc

@@ -270,40 +270,123 @@ error_code ServerFamily::LoadRdb(const std::string& rdb_file) {
   return ec;
 }
 
-void AppendMetric(http::StringResponse* resp, absl::AlphaNum name, absl::AlphaNum val) {
+enum MetricType { COUNTER, GAUGE, SUMMARY, HISTOGRAM };
-  /**
-   * Gets a metric name and a value and write it using Prometheus format.
-   *
-   * TODO:
-   *  1. add metrics descriptions.
-   *  2. support other types of metrics, not just gauge :)
-   */
 
-  const auto full_name = StrCat("dragonfly_", name);
+const char* MetricTypeName(MetricType type) {
-  absl::StrAppend(&resp->body(), "# HELP ", full_name, " ", name, "\n");
+  switch (type) {
-  absl::StrAppend(&resp->body(), "# TYPE ", full_name, " gauge\n");
+    case MetricType::COUNTER:
-  absl::StrAppend(&resp->body(), full_name, " ", val, "\n");
+      return "counter";
+    case MetricType::GAUGE:
+      return "gauge";
+    case MetricType::SUMMARY:
+      return "summary";
+    case MetricType::HISTOGRAM:
+      return "histogram";
+  }
+  return "unknown";
+}
+
+inline string GetMetricFullName(string_view metric_name) {
+  return StrCat("dragonfly_", metric_name);
+}
+
+void AppendMetricHeader(string_view metric_name, string_view metric_help, MetricType type,
+                        string* dest) {
+  const auto full_metric_name = GetMetricFullName(metric_name);
+  absl::StrAppend(dest, "# HELP ", full_metric_name, " ", metric_help, "\n");
+  absl::StrAppend(dest, "# TYPE ", full_metric_name, " ", MetricTypeName(type), "\n");
+}
+
+void AppendLabelTupple(absl::Span<const string_view> label_names,
+                       absl::Span<const string_view> label_values, string* dest) {
+  if (label_names.empty())
+    return;
+
+  absl::StrAppend(dest, "{");
+  for (size_t i = 0; i < label_names.size(); ++i) {
+    if (i > 0) {
+      absl::StrAppend(dest, ", ");
+    }
+    absl::StrAppend(dest, label_names[i], "=\"", label_values[i], "\"");
+  }
+
+  absl::StrAppend(dest, "}");
+}
+
+void AppendMetricValue(string_view metric_name, const absl::AlphaNum& value,
+                       absl::Span<const string_view> label_names,
+                       absl::Span<const string_view> label_values, string* dest) {
+  absl::StrAppend(dest, GetMetricFullName(metric_name));
+  AppendLabelTupple(label_names, label_values, dest);
+  absl::StrAppend(dest, " ", value, "\n");
+}
+
+void AppendMetricWithoutLabels(string_view name, string_view help, const absl::AlphaNum& value,
+                               MetricType type, string* dest) {
+  AppendMetricHeader(name, help, type, dest);
+  AppendMetricValue(name, value, {}, {}, dest);
+}
+
+void PrintPrometheusMetrics(const Metrics& m, http::StringResponse* resp) {
+  // Server metrics
+  AppendMetricWithoutLabels("up", "", 1, MetricType::GAUGE, &resp->body());
+  AppendMetricWithoutLabels("uptime_in_seconds", "", m.uptime, MetricType::GAUGE, &resp->body());
+
+  // Clients metrics
+  AppendMetricWithoutLabels("connected_clients", "", m.conn_stats.num_conns, MetricType::GAUGE,
+                            &resp->body());
+  AppendMetricWithoutLabels("client_read_buf_capacity", "", m.conn_stats.read_buf_capacity,
+                            MetricType::GAUGE, &resp->body());
+  AppendMetricWithoutLabels("blocked_clients", "", m.conn_stats.num_blocked_clients,
+                            MetricType::GAUGE, &resp->body());
+
+  // Memory metrics
+  AppendMetricWithoutLabels("memory_used_bytes", "", m.heap_used_bytes, MetricType::GAUGE, &resp->body());
+  AppendMetricWithoutLabels("memory_used_peak_bytes", "", used_mem_peak.load(memory_order_relaxed),
+                            MetricType::GAUGE, &resp->body());
+  AppendMetricWithoutLabels("comitted_memory", "", _mi_stats_main.committed.current,
+                            MetricType::GAUGE, &resp->body());
+  AppendMetricWithoutLabels("memory_max_bytes", "", max_memory_limit, MetricType::GAUGE, &resp->body());
+
+  AppendMetricWithoutLabels("commands_processed_total", "", m.conn_stats.command_cnt,
+                            MetricType::COUNTER, &resp->body());
+
+  // Net metrics
+  AppendMetricWithoutLabels("net_input_bytes_total", "", m.conn_stats.io_read_bytes,
+                            MetricType::COUNTER, &resp->body());
+  AppendMetricWithoutLabels("net_output_bytes_total", "", m.conn_stats.io_write_bytes,
+                            MetricType::COUNTER, &resp->body());
+
+  // DB stats
+  AppendMetricWithoutLabels("expired_keys_total", "", m.events.expired_keys, MetricType::COUNTER,
+                            &resp->body());
+  AppendMetricWithoutLabels("evicted_keys_total", "", m.events.evicted_keys, MetricType::COUNTER,
+                            &resp->body());
+
+  string db_key_metrics;
+  string db_key_expire_metrics;
+
+  AppendMetricHeader("db_keys", "Total number of keys by DB", MetricType::GAUGE, &db_key_metrics);
+  AppendMetricHeader("db_keys_expiring", "Total number of expiring keys by DB", MetricType::GAUGE,
+                     &db_key_expire_metrics);
+
+  for (size_t i = 0; i < m.db.size(); ++i) {
+    AppendMetricValue("db_keys", m.db[i].key_count, {"db"}, {StrCat("db", i)}, &db_key_metrics);
+    AppendMetricValue("db_keys_expiring", m.db[i].expire_count, {"db"}, {StrCat("db", i)},
+                      &db_key_expire_metrics);
+  }
+
+  absl::StrAppend(&resp->body(), db_key_metrics);
+  absl::StrAppend(&resp->body(), db_key_expire_metrics);
 }
 
 void ServerFamily::ConfigureMetrics(util::HttpListenerBase* http_base) {
-  // The naming of the metrics should be compatible with redis_exporter, see https://github.com/oliver006/redis_exporter/blob/master/exporter/exporter.go#L111
+  // The naming of the metrics should be compatible with redis_exporter, see
+  // https://github.com/oliver006/redis_exporter/blob/master/exporter/exporter.go#L111
 
   auto cb = [this](const http::QueryArgs& args, HttpContext* send) {
     http::StringResponse resp = http::MakeStringResponse(boost::beast::http::status::ok);
-    Metrics m = this->GetMetrics();
+    PrintPrometheusMetrics(this->GetMetrics(), &resp);
-
-    // Server metrics
-    AppendMetric(&resp, "uptime_in_seconds", m.uptime);
-
-    // Clients metrics
-    AppendMetric(&resp, "connected_clients", m.conn_stats.num_conns);
-    AppendMetric(&resp, "client_read_buf_capacity", m.conn_stats.read_buf_capacity);
-    AppendMetric(&resp, "blocked_clients", m.conn_stats.num_blocked_clients);
-
-    // Memory metrics
-    AppendMetric(&resp, "used_memory", m.heap_used_bytes);
-    AppendMetric(&resp, "used_memory_peak", used_mem_peak.load(memory_order_relaxed));
-    AppendMetric(&resp, "comitted_memory", _mi_stats_main.committed.current);
 
     return send->Invoke(std::move(resp));
   };