Add tiered storage stats

src/server/common.cc

@@ -111,4 +111,15 @@ bool ParseHumanReadableBytes(std::string_view str, int64_t* num_bytes) {
   return true;
 }
 
+#define ADD(x) (x) += o.x
+
+TieredStats& TieredStats::operator+=(const TieredStats& o) {
+  static_assert(sizeof(TieredStats) == 16);
+
+  ADD(external_reads);
+  ADD(external_writes);
+
+  return *this;
+}
+
 }  // namespace dfly

src/server/common.h

@@ -58,6 +58,15 @@ struct OpArgs {
   DbIndex db_ind;
 };
 
+
+struct TieredStats {
+  size_t external_reads = 0;
+  size_t external_writes = 0;
+
+  TieredStats& operator+=(const TieredStats&);
+};
+
+
 inline void ToUpper(const MutableSlice* val) {
   for (auto& c : *val) {
     c = absl::ascii_toupper(c);

src/server/db_slice.cc

@@ -87,7 +87,7 @@ class PrimeEvictionPolicy {
 #define ADD(x) (x) += o.x
 
 DbStats& DbStats::operator+=(const DbStats& o) {
-  static_assert(sizeof(DbStats) == 72);
+  static_assert(sizeof(DbStats) == 80);
 
   ADD(key_count);
   ADD(expire_count);
@@ -99,6 +99,7 @@ DbStats& DbStats::operator+=(const DbStats& o) {
   ADD(small_string_bytes);
   ADD(listpack_blob_cnt);
   ADD(listpack_bytes);
+  ADD(external_entries);
 
   return *this;
 }
@@ -154,6 +155,7 @@ auto DbSlice::GetStats() const -> Stats {
     s.db.table_mem_usage += (db->prime_table.mem_usage() + db->expire_table.mem_usage());
     s.db.listpack_blob_cnt += db->stats.listpack_blob_cnt;
     s.db.listpack_bytes += db->stats.listpack_bytes;
+    s.db.external_entries += db->stats.external_entries;
   }
   s.db.small_string_bytes = CompactObj::GetStats().small_string_bytes;
 
@@ -324,7 +326,7 @@ size_t DbSlice::FlushDb(DbIndex db_ind) {
     db->prime_table.Clear();
     db->expire_table.Clear();
     db->mcflag_table.Clear();
-    db->stats = InternalDbStats{};
+    db->stats = PerDbStats{};
 
     return removed;
   };

src/server/db_slice.h

@@ -44,6 +44,9 @@ struct DbStats {
 
   size_t listpack_blob_cnt = 0;
   size_t listpack_bytes = 0;
+
+  size_t external_entries = 0;
+
   DbStats& operator+=(const DbStats& o);
 };
 
@@ -67,7 +70,7 @@ class DbSlice {
     SliceEvents events;
   };
 
-  struct InternalDbStats {
+  struct PerDbStats {
     // Number of inline keys.
     uint64_t inline_keys = 0;
 
@@ -76,6 +79,7 @@ class DbSlice {
     size_t obj_memory_usage = 0;
     size_t listpack_blob_cnt = 0;
     size_t listpack_bytes = 0;
+    size_t external_entries = 0;
   };
 
 
@@ -195,7 +199,7 @@ class DbSlice {
   void PreUpdate(DbIndex db_ind, PrimeIterator it);
   void PostUpdate(DbIndex db_ind, PrimeIterator it);
 
-  InternalDbStats* MutableStats(DbIndex db_ind) {
+  PerDbStats* MutableStats(DbIndex db_ind) {
     return &db_arr_[db_ind]->stats;
   }
 
@@ -251,7 +255,7 @@ class DbSlice {
 
     LockTable lock_table;
 
-    mutable InternalDbStats stats;
+    mutable PerDbStats stats;
 
     explicit DbWrapper(std::pmr::memory_resource* mr);
   };

src/server/hset_family.cc

@@ -443,7 +443,7 @@ OpResult<uint32_t> HSetFamily::OpSet(const OpArgs& op_args, string_view key, Cmd
   auto& db_slice = op_args.shard->db_slice();
   const auto [it, inserted] = db_slice.AddOrFind(op_args.db_ind, key);
 
-  DbSlice::InternalDbStats* stats = db_slice.MutableStats(op_args.db_ind);
+  DbSlice::PerDbStats* stats = db_slice.MutableStats(op_args.db_ind);
 
   robj* hset = nullptr;
   uint8_t* lp = nullptr;
@@ -524,7 +524,7 @@ OpResult<uint32_t> HSetFamily::OpDel(const OpArgs& op_args, string_view key, Cmd
   robj* hset = co.AsRObj();
   unsigned deleted = 0;
   bool key_remove = false;
-  DbSlice::InternalDbStats* stats = db_slice.MutableStats(op_args.db_ind);
+  DbSlice::PerDbStats* stats = db_slice.MutableStats(op_args.db_ind);
 
   if (hset->encoding == OBJ_ENCODING_LISTPACK) {
     stats->listpack_bytes -= lpBytes((uint8_t*)hset->ptr);
@@ -753,7 +753,7 @@ OpStatus HSetFamily::OpIncrBy(const OpArgs& op_args, string_view key, string_vie
   auto& db_slice = op_args.shard->db_slice();
   const auto [it, inserted] = db_slice.AddOrFind(op_args.db_ind, key);
 
-  DbSlice::InternalDbStats* stats = db_slice.MutableStats(op_args.db_ind);
+  DbSlice::PerDbStats* stats = db_slice.MutableStats(op_args.db_ind);
 
   robj* hset = nullptr;
   size_t lpb = 0;

src/server/server_family.cc

@@ -28,6 +28,7 @@ extern "C" {
 #include "server/replica.h"
 #include "server/script_mgr.h"
 #include "server/server_state.h"
+#include "server/tiered_storage.h"
 #include "server/transaction.h"
 #include "strings/human_readable.h"
 #include "util/accept_server.h"
@@ -401,6 +402,9 @@ Metrics ServerFamily::GetMetrics() const {
       result.events += db_stats.events;
 
       result.heap_used_bytes += shard->UsedMemory();
+      if (shard->tiered_storage()) {
+        result.tiered_stats += shard->tiered_storage()->stats();
+      }
     }
   };
 
@@ -514,6 +518,13 @@ tcp_port:)";
     append("async_writes_count:", m.conn_stats.async_writes_cnt);
   }
 
+  if (should_enter("TIERED", true)) {
+    ADD_HEADER("# TIERED_STORAGE");
+    append("external_entries:", m.db.external_entries);
+    append("external_reads:", m.tiered_stats.external_reads);
+    append("external_writes:", m.tiered_stats.external_writes);
+  }
+
   if (should_enter("REPLICATION")) {
     ADD_HEADER("# Replication");
 

src/server/server_family.h

@@ -25,6 +25,7 @@ class ScriptMgr;
 struct Metrics {
   DbStats db;
   SliceEvents events;
+  TieredStats tiered_stats;
 
   size_t qps = 0;
   size_t heap_used_bytes = 0;

src/server/tiered_storage.cc

@@ -75,6 +75,7 @@ void TieredStorage::SendIoRequest(size_t offset, size_t req_size, ActiveIoReques
 
   auto cb = [this, req](int res) { FinishIoRequest(res, req); };
   io_mgr_.WriteAsync(offset, sv, move(cb));
+  ++stats_.external_writes;
 #else
   FinishIoRequest(0, req);
 #endif
@@ -98,6 +99,7 @@ void TieredStorage::FinishIoRequest(int io_res, ActiveIoRequest* req) {
     if (success) {
       size_t item_size = it->second.Size();
       it->second.SetExternal(k_v.second, item_size);
+      ++db_slice_.MutableStats(ikey.db_indx)->external_entries;
     }
   }
   --num_active_requests_;
@@ -124,6 +126,12 @@ error_code TieredStorage::Open(const string& path) {
   return ec;
 }
 
+std::error_code TieredStorage::Read(size_t offset, size_t len, char* dest) {
+  stats_.external_reads++;
+
+  return io_mgr_.Read(offset, io::MutableBytes{reinterpret_cast<uint8_t*>(dest), len});
+}
+
 void TieredStorage::Shutdown() {
   io_mgr_.Shutdown();
 }

src/server/tiered_storage.h

@@ -21,14 +21,16 @@ class TieredStorage {
 
   std::error_code Open(const std::string& path);
 
-  std::error_code Read(size_t offset, size_t len, char* dest) {
-    return io_mgr_.Read(offset, io::MutableBytes{reinterpret_cast<uint8_t*>(dest), len});
-  }
+  std::error_code Read(size_t offset, size_t len, char* dest);
 
   std::error_code UnloadItem(DbIndex db_index, PrimeIterator it);
 
   void Shutdown();
 
+  const TieredStats& stats() const {
+    return stats_;
+  }
+
  private:
   struct ActiveIoRequest;
 
@@ -38,7 +40,6 @@ class TieredStorage {
   void SendIoRequest(size_t offset, size_t req_size, ActiveIoRequest* req);
   void FinishIoRequest(int io_res, ActiveIoRequest* req);
 
-
   DbSlice& db_slice_;
   IoMgr io_mgr_;
   ExternalAllocator alloc_;
@@ -61,6 +62,7 @@ class TieredStorage {
   };
 
   std::vector<PerDb*> db_arr_;
+  TieredStats stats_;
 };
 
 }  // namespace dfly