From 8d312a92e1e1f71ec44ec91222b89c53913f238e Mon Sep 17 00:00:00 2001 From: Roman Gershman Date: Fri, 22 Apr 2022 18:07:35 +0300 Subject: [PATCH] Add tiered storage stats --- src/server/common.cc | 11 +++++++++++ src/server/common.h | 9 +++++++++ src/server/db_slice.cc | 6 ++++-- src/server/db_slice.h | 10 +++++++--- src/server/hset_family.cc | 6 +++--- src/server/server_family.cc | 11 +++++++++++ src/server/server_family.h | 1 + src/server/tiered_storage.cc | 8 ++++++++ src/server/tiered_storage.h | 10 ++++++---- 9 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/server/common.cc b/src/server/common.cc index 62358c2..bf87f9d 100644 --- a/src/server/common.cc +++ b/src/server/common.cc @@ -111,4 +111,15 @@ bool ParseHumanReadableBytes(std::string_view str, int64_t* num_bytes) { return true; } +#define ADD(x) (x) += o.x + +TieredStats& TieredStats::operator+=(const TieredStats& o) { + static_assert(sizeof(TieredStats) == 16); + + ADD(external_reads); + ADD(external_writes); + + return *this; +} + } // namespace dfly diff --git a/src/server/common.h b/src/server/common.h index 78b9b3c..f95c882 100644 --- a/src/server/common.h +++ b/src/server/common.h @@ -58,6 +58,15 @@ struct OpArgs { DbIndex db_ind; }; + +struct TieredStats { + size_t external_reads = 0; + size_t external_writes = 0; + + TieredStats& operator+=(const TieredStats&); +}; + + inline void ToUpper(const MutableSlice* val) { for (auto& c : *val) { c = absl::ascii_toupper(c); diff --git a/src/server/db_slice.cc b/src/server/db_slice.cc index a865ae1..bbcc6fc 100644 --- a/src/server/db_slice.cc +++ b/src/server/db_slice.cc @@ -87,7 +87,7 @@ class PrimeEvictionPolicy { #define ADD(x) (x) += o.x DbStats& DbStats::operator+=(const DbStats& o) { - static_assert(sizeof(DbStats) == 72); + static_assert(sizeof(DbStats) == 80); ADD(key_count); ADD(expire_count); @@ -99,6 +99,7 @@ DbStats& DbStats::operator+=(const DbStats& o) { ADD(small_string_bytes); ADD(listpack_blob_cnt); ADD(listpack_bytes); + ADD(external_entries); return *this; } @@ -154,6 +155,7 @@ auto DbSlice::GetStats() const -> Stats { s.db.table_mem_usage += (db->prime_table.mem_usage() + db->expire_table.mem_usage()); s.db.listpack_blob_cnt += db->stats.listpack_blob_cnt; s.db.listpack_bytes += db->stats.listpack_bytes; + s.db.external_entries += db->stats.external_entries; } s.db.small_string_bytes = CompactObj::GetStats().small_string_bytes; @@ -324,7 +326,7 @@ size_t DbSlice::FlushDb(DbIndex db_ind) { db->prime_table.Clear(); db->expire_table.Clear(); db->mcflag_table.Clear(); - db->stats = InternalDbStats{}; + db->stats = PerDbStats{}; return removed; }; diff --git a/src/server/db_slice.h b/src/server/db_slice.h index 67ca720..53ae8c4 100644 --- a/src/server/db_slice.h +++ b/src/server/db_slice.h @@ -44,6 +44,9 @@ struct DbStats { size_t listpack_blob_cnt = 0; size_t listpack_bytes = 0; + + size_t external_entries = 0; + DbStats& operator+=(const DbStats& o); }; @@ -67,7 +70,7 @@ class DbSlice { SliceEvents events; }; - struct InternalDbStats { + struct PerDbStats { // Number of inline keys. uint64_t inline_keys = 0; @@ -76,6 +79,7 @@ class DbSlice { size_t obj_memory_usage = 0; size_t listpack_blob_cnt = 0; size_t listpack_bytes = 0; + size_t external_entries = 0; }; @@ -195,7 +199,7 @@ class DbSlice { void PreUpdate(DbIndex db_ind, PrimeIterator it); void PostUpdate(DbIndex db_ind, PrimeIterator it); - InternalDbStats* MutableStats(DbIndex db_ind) { + PerDbStats* MutableStats(DbIndex db_ind) { return &db_arr_[db_ind]->stats; } @@ -251,7 +255,7 @@ class DbSlice { LockTable lock_table; - mutable InternalDbStats stats; + mutable PerDbStats stats; explicit DbWrapper(std::pmr::memory_resource* mr); }; diff --git a/src/server/hset_family.cc b/src/server/hset_family.cc index 947ab31..5aaddd4 100644 --- a/src/server/hset_family.cc +++ b/src/server/hset_family.cc @@ -443,7 +443,7 @@ OpResult HSetFamily::OpSet(const OpArgs& op_args, string_view key, Cmd auto& db_slice = op_args.shard->db_slice(); const auto [it, inserted] = db_slice.AddOrFind(op_args.db_ind, key); - DbSlice::InternalDbStats* stats = db_slice.MutableStats(op_args.db_ind); + DbSlice::PerDbStats* stats = db_slice.MutableStats(op_args.db_ind); robj* hset = nullptr; uint8_t* lp = nullptr; @@ -524,7 +524,7 @@ OpResult HSetFamily::OpDel(const OpArgs& op_args, string_view key, Cmd robj* hset = co.AsRObj(); unsigned deleted = 0; bool key_remove = false; - DbSlice::InternalDbStats* stats = db_slice.MutableStats(op_args.db_ind); + DbSlice::PerDbStats* stats = db_slice.MutableStats(op_args.db_ind); if (hset->encoding == OBJ_ENCODING_LISTPACK) { stats->listpack_bytes -= lpBytes((uint8_t*)hset->ptr); @@ -753,7 +753,7 @@ OpStatus HSetFamily::OpIncrBy(const OpArgs& op_args, string_view key, string_vie auto& db_slice = op_args.shard->db_slice(); const auto [it, inserted] = db_slice.AddOrFind(op_args.db_ind, key); - DbSlice::InternalDbStats* stats = db_slice.MutableStats(op_args.db_ind); + DbSlice::PerDbStats* stats = db_slice.MutableStats(op_args.db_ind); robj* hset = nullptr; size_t lpb = 0; diff --git a/src/server/server_family.cc b/src/server/server_family.cc index 47fce24..4701124 100644 --- a/src/server/server_family.cc +++ b/src/server/server_family.cc @@ -28,6 +28,7 @@ extern "C" { #include "server/replica.h" #include "server/script_mgr.h" #include "server/server_state.h" +#include "server/tiered_storage.h" #include "server/transaction.h" #include "strings/human_readable.h" #include "util/accept_server.h" @@ -401,6 +402,9 @@ Metrics ServerFamily::GetMetrics() const { result.events += db_stats.events; result.heap_used_bytes += shard->UsedMemory(); + if (shard->tiered_storage()) { + result.tiered_stats += shard->tiered_storage()->stats(); + } } }; @@ -514,6 +518,13 @@ tcp_port:)"; append("async_writes_count:", m.conn_stats.async_writes_cnt); } + if (should_enter("TIERED", true)) { + ADD_HEADER("# TIERED_STORAGE"); + append("external_entries:", m.db.external_entries); + append("external_reads:", m.tiered_stats.external_reads); + append("external_writes:", m.tiered_stats.external_writes); + } + if (should_enter("REPLICATION")) { ADD_HEADER("# Replication"); diff --git a/src/server/server_family.h b/src/server/server_family.h index 199dd7e..30df823 100644 --- a/src/server/server_family.h +++ b/src/server/server_family.h @@ -25,6 +25,7 @@ class ScriptMgr; struct Metrics { DbStats db; SliceEvents events; + TieredStats tiered_stats; size_t qps = 0; size_t heap_used_bytes = 0; diff --git a/src/server/tiered_storage.cc b/src/server/tiered_storage.cc index 9404fba..44ec6ed 100644 --- a/src/server/tiered_storage.cc +++ b/src/server/tiered_storage.cc @@ -75,6 +75,7 @@ void TieredStorage::SendIoRequest(size_t offset, size_t req_size, ActiveIoReques auto cb = [this, req](int res) { FinishIoRequest(res, req); }; io_mgr_.WriteAsync(offset, sv, move(cb)); + ++stats_.external_writes; #else FinishIoRequest(0, req); #endif @@ -98,6 +99,7 @@ void TieredStorage::FinishIoRequest(int io_res, ActiveIoRequest* req) { if (success) { size_t item_size = it->second.Size(); it->second.SetExternal(k_v.second, item_size); + ++db_slice_.MutableStats(ikey.db_indx)->external_entries; } } --num_active_requests_; @@ -124,6 +126,12 @@ error_code TieredStorage::Open(const string& path) { return ec; } +std::error_code TieredStorage::Read(size_t offset, size_t len, char* dest) { + stats_.external_reads++; + + return io_mgr_.Read(offset, io::MutableBytes{reinterpret_cast(dest), len}); +} + void TieredStorage::Shutdown() { io_mgr_.Shutdown(); } diff --git a/src/server/tiered_storage.h b/src/server/tiered_storage.h index 25ac1a4..092d866 100644 --- a/src/server/tiered_storage.h +++ b/src/server/tiered_storage.h @@ -21,14 +21,16 @@ class TieredStorage { std::error_code Open(const std::string& path); - std::error_code Read(size_t offset, size_t len, char* dest) { - return io_mgr_.Read(offset, io::MutableBytes{reinterpret_cast(dest), len}); - } + std::error_code Read(size_t offset, size_t len, char* dest); std::error_code UnloadItem(DbIndex db_index, PrimeIterator it); void Shutdown(); + const TieredStats& stats() const { + return stats_; + } + private: struct ActiveIoRequest; @@ -38,7 +40,6 @@ class TieredStorage { void SendIoRequest(size_t offset, size_t req_size, ActiveIoRequest* req); void FinishIoRequest(int io_res, ActiveIoRequest* req); - DbSlice& db_slice_; IoMgr io_mgr_; ExternalAllocator alloc_; @@ -61,6 +62,7 @@ class TieredStorage { }; std::vector db_arr_; + TieredStats stats_; }; } // namespace dfly