opensci
/
gStore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

<feat> <add Vlist's cache in preID2values> 

I added Vlist's cache in preID2values. Notice the members' name, they mey be confusing.
This commit is contained in:
qinzongyue 2017-07-15 15:12:32 +08:00
parent b51281dedf
commit 45e87ce393
4 changed files with 115 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
Database/Database.cpp
View File

@ -683,6 +683,7 @@ Database::load_cache()
for(int i = 0; i < important_preID.size(); ++i)
cout << important_preID[i] << ' ';
cout << endl;
this->load_candidate_pre2values();
this->load_important_sub2values();
this->load_important_obj2values();
}
@ -718,6 +719,92 @@ Database::load_important_sub2values()
this->build_CacheOfSub2values();
}
void
Database::load_candidate_pre2values()
{
cout << "get candidate preID..." << endl;
this->get_candidate_preID();
this->build_CacheOfPre2values();
}
void
Database::get_candidate_preID()
{
//cout << "now add cache of preID2values..." << endl;
/*for(int i = 0; i < important_preID.size(); ++i)
{
unsigned _size = this->kvstore->getPreListSize(important_preID[i]);
if (now_size + _size >= max_total_size) continue;
now_size += _size;
this->kvstore->AddIntoPreCache(important_preID[i]);
}*/
unsigned now_total_size = 0;
const unsigned max_total_size = 2000000000;//2G
// std::priority_queue <KEY_SIZE_VALUE> candidate_preID;
std::priority_queue <KEY_SIZE_VALUE, deque<KEY_SIZE_VALUE>, greater<KEY_SIZE_VALUE> > rubbish;
while(!rubbish.empty()) rubbish.pop();
while(!candidate_preID.empty()) candidate_preID.pop();
for(TYPE_PREDICATE_ID i = 0; i <= limitID_predicate; ++i)
{
unsigned _value = 0;
unsigned _size;
_size = this->kvstore->getPreListSize(i);
if (!VList::isLongList(_size)) continue; // only long list need to be stored in cache
_value = pre2num[i];
if (_size + now_total_size < max_total_size)
{
candidate_preID.push(KEY_SIZE_VALUE(i, _size, _value));
now_total_size += _size;
}
else
{
if (candidate_preID.empty()) continue;
if (_value > candidate_preID.top().value)
{
while (now_total_size + _size >= max_total_size)
{
if (candidate_preID.top().value >= _value) break;
rubbish.push(candidate_preID.top());
now_total_size -= candidate_preID.top().size;
candidate_preID.pop();
}
if (now_total_size + _size < max_total_size)
{
now_total_size += _size;
candidate_preID.push(KEY_SIZE_VALUE(i, _size, _value));
}
while (!rubbish.empty())
{
if (now_total_size + rubbish.top().size < max_total_size)
{
now_total_size += rubbish.top().size;
candidate_preID.push(rubbish.top());
}
rubbish.pop();
}
}
}
}
cout << "finish getting candidate preID, the size is " << now_total_size << endl;
}
void
Database::build_CacheOfPre2values()
{
cout << "now add cache of preID2values..." << endl;
while (!candidate_preID.empty())
{
//cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl;
this->kvstore->AddIntoPreCache(candidate_preID.top().key);
candidate_preID.pop();
}
}
void
Database::build_CacheOfObj2values()
{
@ -747,7 +834,7 @@ Database::get_important_subID()
{
while(!important_subID.empty()) important_subID.pop();
unsigned now_total_size = 0;
const unsigned max_total_size = 500000000;//0.5G
const unsigned max_total_size = 2000000000;//2G
std::priority_queue <KEY_SIZE_VALUE, deque<KEY_SIZE_VALUE>, greater<KEY_SIZE_VALUE> > rubbish;
while(!rubbish.empty()) rubbish.pop();
// a sub who has largest degree with important pre is important subs
@ -755,7 +842,6 @@ Database::get_important_subID()
{
unsigned _value = 0;
unsigned _size;
unsigned* _tmp = NULL;
_size = this->kvstore->getSubListSize(i);
if (!VList::isLongList(_size)) continue; // only long list need to be stored in cache
@ -771,7 +857,8 @@ Database::get_important_subID()
}
else
{
if (!important_subID.empty() && _value > important_subID.top().value)
if (important_subID.empty()) continue;
if (_value > important_subID.top().value)
{
while (now_total_size + _size >= max_total_size)
{
@ -805,7 +892,7 @@ Database::get_important_objID()
{
while(!important_objID.empty()) important_objID.pop();
unsigned now_total_size = 0;
const unsigned max_total_size = 1500000000;//1.5G
const unsigned max_total_size = 2000000000;//2G
std::priority_queue <KEY_SIZE_VALUE, deque<KEY_SIZE_VALUE>, greater<KEY_SIZE_VALUE> > rubbish;
while(!rubbish.empty()) rubbish.pop();
// a sub who has largest degree with important pre is important subs
@ -813,7 +900,6 @@ Database::get_important_objID()
{
unsigned _value = 0;
unsigned _size;
unsigned* _tmp = NULL;
_size = this->kvstore->getObjListSize(i);
if (!VList::isLongList(_size)) continue; // only long list need to be stored in cache
@ -830,7 +916,8 @@ Database::get_important_objID()
}
else
{
if (!important_objID.empty() && _value > important_objID.top().value)
if (important_objID.empty()) continue;
if (_value > important_objID.top().value)
{
while (now_total_size + _size >= max_total_size)
{

3
Database/Database.h
View File

@ -157,11 +157,14 @@ private:
std::vector <TYPE_PREDICATE_ID> important_preID;
void load_important_sub2values();
void load_important_obj2values();
void load_candidate_pre2values();
void build_CacheOfPre2values();
void build_CacheOfSub2values();
void build_CacheOfObj2values();
void get_important_subID();
void get_important_objID();
void get_candidate_preID();
std::priority_queue <KEY_SIZE_VALUE> candidate_preID;
std::priority_queue <KEY_SIZE_VALUE> important_subID;
std::priority_queue <KEY_SIZE_VALUE> important_objID;

17
KVstore/KVstore.cpp
View File

@ -2499,6 +2499,11 @@ KVstore::isEntity(TYPE_ENTITY_LITERAL_ID id)
return id < Util::LITERAL_FIRST_ID;
}
void
KVstore::AddIntoPreCache(TYPE_PREDICATE_ID _pre_id)
{
this->preID2values->AddIntoCache(_pre_id);
}
void
KVstore::AddIntoSubCache(TYPE_ENTITY_LITERAL_ID _entity_id)
{
@ -2517,6 +2522,7 @@ KVstore::getSubListSize(TYPE_ENTITY_LITERAL_ID _sub_id)
unsigned* _tmp = NULL;
unsigned _ret;
this->getValueByKey(this->subID2values, _sub_id, (char*&) _tmp, _ret);
delete [] _tmp;
return _ret;
}
@ -2526,6 +2532,17 @@ KVstore::getObjListSize(TYPE_ENTITY_LITERAL_ID _obj_id)
unsigned* _tmp = NULL;
unsigned _ret;
this->getValueByKey(this->objID2values, _obj_id, (char*&) _tmp, _ret);
delete [] _tmp;
return _ret;
}
unsigned
KVstore::getPreListSize(TYPE_PREDICATE_ID _pre_id)
{
unsigned* _tmp = NULL;
unsigned _ret;
this->getValueByKey(this->preID2values, _pre_id, (char*&) _tmp, _ret);
delete [] _tmp;
return _ret;
}

2
KVstore/KVstore.h
View File

@ -52,6 +52,8 @@ public:
// functions to load Vlist's cache
void AddIntoSubCache(TYPE_ENTITY_LITERAL_ID _entity_id);
void AddIntoObjCache(TYPE_ENTITY_LITERAL_ID _entity_literal_id);
void AddIntoPreCache(TYPE_PREDICATE_ID _pre_id);
unsigned getPreListSize(TYPE_PREDICATE_ID _pre_id);
unsigned getSubListSize(TYPE_ENTITY_LITERAL_ID _sub_id);
unsigned getObjListSize(TYPE_ENTITY_LITERAL_ID _obj_id);