diff --git a/Database/Database.cpp b/Database/Database.cpp index 19a3848..1238338 100644 --- a/Database/Database.cpp +++ b/Database/Database.cpp @@ -683,6 +683,7 @@ Database::load_cache() for(int i = 0; i < important_preID.size(); ++i) cout << important_preID[i] << ' '; cout << endl; + this->load_candidate_pre2values(); this->load_important_sub2values(); this->load_important_obj2values(); } @@ -718,6 +719,92 @@ Database::load_important_sub2values() this->build_CacheOfSub2values(); } +void +Database::load_candidate_pre2values() +{ + cout << "get candidate preID..." << endl; + this->get_candidate_preID(); + + this->build_CacheOfPre2values(); +} + +void +Database::get_candidate_preID() +{ + //cout << "now add cache of preID2values..." << endl; + /*for(int i = 0; i < important_preID.size(); ++i) + { + unsigned _size = this->kvstore->getPreListSize(important_preID[i]); + if (now_size + _size >= max_total_size) continue; + now_size += _size; + this->kvstore->AddIntoPreCache(important_preID[i]); + }*/ + unsigned now_total_size = 0; + const unsigned max_total_size = 2000000000;//2G +// std::priority_queue candidate_preID; + std::priority_queue , greater > rubbish; + while(!rubbish.empty()) rubbish.pop(); + while(!candidate_preID.empty()) candidate_preID.pop(); + for(TYPE_PREDICATE_ID i = 0; i <= limitID_predicate; ++i) + { + unsigned _value = 0; + unsigned _size; + + _size = this->kvstore->getPreListSize(i); + + if (!VList::isLongList(_size)) continue; // only long list need to be stored in cache + + _value = pre2num[i]; + + if (_size + now_total_size < max_total_size) + { + candidate_preID.push(KEY_SIZE_VALUE(i, _size, _value)); + now_total_size += _size; + } + else + { + if (candidate_preID.empty()) continue; + if (_value > candidate_preID.top().value) + { + while (now_total_size + _size >= max_total_size) + { + if (candidate_preID.top().value >= _value) break; + rubbish.push(candidate_preID.top()); + now_total_size -= candidate_preID.top().size; + candidate_preID.pop(); + } + if (now_total_size + _size < max_total_size) + { + now_total_size += _size; + candidate_preID.push(KEY_SIZE_VALUE(i, _size, _value)); + } + while (!rubbish.empty()) + { + if (now_total_size + rubbish.top().size < max_total_size) + { + now_total_size += rubbish.top().size; + candidate_preID.push(rubbish.top()); + } + rubbish.pop(); + } + } + } + } + cout << "finish getting candidate preID, the size is " << now_total_size << endl; +} + +void +Database::build_CacheOfPre2values() +{ + cout << "now add cache of preID2values..." << endl; + while (!candidate_preID.empty()) + { + //cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl; + this->kvstore->AddIntoPreCache(candidate_preID.top().key); + candidate_preID.pop(); + } +} + void Database::build_CacheOfObj2values() { @@ -747,7 +834,7 @@ Database::get_important_subID() { while(!important_subID.empty()) important_subID.pop(); unsigned now_total_size = 0; - const unsigned max_total_size = 500000000;//0.5G + const unsigned max_total_size = 2000000000;//2G std::priority_queue , greater > rubbish; while(!rubbish.empty()) rubbish.pop(); // a sub who has largest degree with important pre is important subs @@ -755,7 +842,6 @@ Database::get_important_subID() { unsigned _value = 0; unsigned _size; - unsigned* _tmp = NULL; _size = this->kvstore->getSubListSize(i); if (!VList::isLongList(_size)) continue; // only long list need to be stored in cache @@ -771,7 +857,8 @@ Database::get_important_subID() } else { - if (!important_subID.empty() && _value > important_subID.top().value) + if (important_subID.empty()) continue; + if (_value > important_subID.top().value) { while (now_total_size + _size >= max_total_size) { @@ -805,7 +892,7 @@ Database::get_important_objID() { while(!important_objID.empty()) important_objID.pop(); unsigned now_total_size = 0; - const unsigned max_total_size = 1500000000;//1.5G + const unsigned max_total_size = 2000000000;//2G std::priority_queue , greater > rubbish; while(!rubbish.empty()) rubbish.pop(); // a sub who has largest degree with important pre is important subs @@ -813,7 +900,6 @@ Database::get_important_objID() { unsigned _value = 0; unsigned _size; - unsigned* _tmp = NULL; _size = this->kvstore->getObjListSize(i); if (!VList::isLongList(_size)) continue; // only long list need to be stored in cache @@ -830,7 +916,8 @@ Database::get_important_objID() } else { - if (!important_objID.empty() && _value > important_objID.top().value) + if (important_objID.empty()) continue; + if (_value > important_objID.top().value) { while (now_total_size + _size >= max_total_size) { diff --git a/Database/Database.h b/Database/Database.h index e85cb9e..9af8f63 100644 --- a/Database/Database.h +++ b/Database/Database.h @@ -157,11 +157,14 @@ private: std::vector important_preID; void load_important_sub2values(); void load_important_obj2values(); + void load_candidate_pre2values(); void build_CacheOfPre2values(); void build_CacheOfSub2values(); void build_CacheOfObj2values(); void get_important_subID(); void get_important_objID(); + void get_candidate_preID(); + std::priority_queue candidate_preID; std::priority_queue important_subID; std::priority_queue important_objID; diff --git a/KVstore/KVstore.cpp b/KVstore/KVstore.cpp index 6ee4256..ae87679 100644 --- a/KVstore/KVstore.cpp +++ b/KVstore/KVstore.cpp @@ -2499,6 +2499,11 @@ KVstore::isEntity(TYPE_ENTITY_LITERAL_ID id) return id < Util::LITERAL_FIRST_ID; } +void +KVstore::AddIntoPreCache(TYPE_PREDICATE_ID _pre_id) +{ + this->preID2values->AddIntoCache(_pre_id); +} void KVstore::AddIntoSubCache(TYPE_ENTITY_LITERAL_ID _entity_id) { @@ -2517,6 +2522,7 @@ KVstore::getSubListSize(TYPE_ENTITY_LITERAL_ID _sub_id) unsigned* _tmp = NULL; unsigned _ret; this->getValueByKey(this->subID2values, _sub_id, (char*&) _tmp, _ret); + delete [] _tmp; return _ret; } @@ -2526,6 +2532,17 @@ KVstore::getObjListSize(TYPE_ENTITY_LITERAL_ID _obj_id) unsigned* _tmp = NULL; unsigned _ret; this->getValueByKey(this->objID2values, _obj_id, (char*&) _tmp, _ret); + delete [] _tmp; + return _ret; +} + +unsigned +KVstore::getPreListSize(TYPE_PREDICATE_ID _pre_id) +{ + unsigned* _tmp = NULL; + unsigned _ret; + this->getValueByKey(this->preID2values, _pre_id, (char*&) _tmp, _ret); + delete [] _tmp; return _ret; } diff --git a/KVstore/KVstore.h b/KVstore/KVstore.h index 1264c34..6d91735 100644 --- a/KVstore/KVstore.h +++ b/KVstore/KVstore.h @@ -52,6 +52,8 @@ public: // functions to load Vlist's cache void AddIntoSubCache(TYPE_ENTITY_LITERAL_ID _entity_id); void AddIntoObjCache(TYPE_ENTITY_LITERAL_ID _entity_literal_id); + void AddIntoPreCache(TYPE_PREDICATE_ID _pre_id); + unsigned getPreListSize(TYPE_PREDICATE_ID _pre_id); unsigned getSubListSize(TYPE_ENTITY_LITERAL_ID _sub_id); unsigned getObjListSize(TYPE_ENTITY_LITERAL_ID _obj_id);