diff --git a/Database/Database.cpp b/Database/Database.cpp index bde52aa..7ee9ba1 100644 --- a/Database/Database.cpp +++ b/Database/Database.cpp @@ -717,7 +717,7 @@ Database::load() //vstree_thread.join(); #endif //load cache of sub2values and obj2values - this->load_cache(); + //this->load_cache(); //warm up always as finishing build(), to utilize the system buffer //this->warmUp(); @@ -867,36 +867,55 @@ void Database::build_CacheOfPre2values() { cout << "now add cache of preID2values..." << endl; + priority_queue , CmpByMod<2000> > temp_queue; while (!candidate_preID.empty()) { - //cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl; - this->kvstore->AddIntoPreCache(candidate_preID.top().key); + temp_queue.push(candidate_preID.top()); candidate_preID.pop(); } + while (!temp_queue.empty()) + { + //cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl; + this->kvstore->AddIntoPreCache(temp_queue.top().key); + temp_queue.pop(); + } } void Database::build_CacheOfObj2values() { cout << "now add cache of objID2values..." << endl; + // sort key according to their mod by 2000 + priority_queue , CmpByMod<2000> > temp_queue; while (!important_objID.empty()) { - //cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl; - this->kvstore->AddIntoObjCache(important_objID.top().key); + temp_queue.push(important_objID.top()); important_objID.pop(); } + while (!temp_queue.empty()) + { + //cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl; + this->kvstore->AddIntoObjCache(temp_queue.top().key); + temp_queue.pop(); + } } void Database::build_CacheOfSub2values() { cout << "now add cache of subID2values..." << endl; + priority_queue , CmpByMod<2000> > temp_queue; while (!important_subID.empty()) { - //cout << "add key " << important_subID.top().key << " size: " << important_subID.top().size << endl; - this->kvstore->AddIntoSubCache(important_subID.top().key); + temp_queue.push(important_subID.top()); important_subID.pop(); } + while (!temp_queue.empty()) + { + //cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl; + this->kvstore->AddIntoSubCache(temp_queue.top().key); + temp_queue.pop(); + } } void diff --git a/KVstore/IVTree/IVTree.cpp b/KVstore/IVTree/IVTree.cpp index 1252177..06c142c 100644 --- a/KVstore/IVTree/IVTree.cpp +++ b/KVstore/IVTree/IVTree.cpp @@ -601,6 +601,7 @@ IVTree::save() //save the whole tree to disk #ifdef DEBUG_KVSTORE printf("now to save tree!\n"); #endif + this->value_list->release_cache(); if (TSM->writeTree(this->root)) return true; else diff --git a/KVstore/IVTree/node/IVLeafNode.cpp b/KVstore/IVTree/node/IVLeafNode.cpp index 0bceee9..4347142 100644 --- a/KVstore/IVTree/node/IVLeafNode.cpp +++ b/KVstore/IVTree/node/IVLeafNode.cpp @@ -157,7 +157,7 @@ IVLeafNode::setValue(VList* _vlist, int _index, char* _str, unsigned _len, bool cout<<"this is a vlist in set()"<values[_index].getLen(); - _vlist->removeValue(block_num); + _vlist->removeValue(block_num, this->keys[_index]); } else { @@ -247,7 +247,7 @@ IVLeafNode::subValue(VList* _vlist, int _index, bool ifdel) if(this->values[_index].isBstrLongList()) { unsigned block_num = this->values[_index].getLen(); - _vlist->removeValue(block_num); + _vlist->removeValue(block_num, this->keys[_index]); } else { diff --git a/Util/ClassForVlistCache.h b/Util/ClassForVlistCache.h index 2e8a5bb..9be0f65 100644 --- a/Util/ClassForVlistCache.h +++ b/Util/ClassForVlistCache.h @@ -26,12 +26,25 @@ public: _len = 0; _str = NULL; } - ~Longlist_inMem() + void free() { if(_str != NULL) { delete [] _str; _str = NULL; } + key = -1; + _len = 0; + } +}; + + +template +class CmpByMod +{ +public: + bool operator () (const KEY_SIZE_VALUE& a, const KEY_SIZE_VALUE& b) + { + return (a.key % mod) > (b.key % mod); } }; diff --git a/Util/VList.cpp b/Util/VList.cpp index 77c2ca6..a58a745 100644 --- a/Util/VList.cpp +++ b/Util/VList.cpp @@ -208,8 +208,8 @@ VList::readValue(unsigned _block_num, char*& _str, unsigned& _len, unsigned _key unsigned node = _key % 2000, i; for(i = node; longlist[i].key % 2000 <= node; ++i) { - if (longlist[i].key == (int)_key || longlist[i].key == -1) break; - if (i > 1999) i %= 2000; + if (longlist[i].key == (int)_key || longlist[i].key == -1 || i == node - 1) break; + if (i > 1999) i -= 2000; } if (longlist[i].key == (int)_key) // value is in cache { @@ -217,9 +217,12 @@ VList::readValue(unsigned _block_num, char*& _str, unsigned& _len, unsigned _key // accessOfCache++; _len = longlist[i]._len; _str = new char[_len]; - memcpy(_str, longlist[i]._str, _len); - // cout << _len << endl; - return true; + if (_str != NULL) + { + memcpy(_str, longlist[i]._str, _len); + // cout << _len << endl; + return true; + } } //if not in cache, read from disk(add a random seek time), the pointer should be clear in upper layer @@ -271,7 +274,7 @@ VList::writeValue(char* _str, unsigned _len) } bool -VList::removeValue(unsigned _block_num) +VList::removeValue(unsigned _block_num, unsigned _key) { CACHE_ITERATOR it = this->vlist_cache.find(_block_num); if(it != this->vlist_cache.end()) @@ -281,6 +284,15 @@ VList::removeValue(unsigned _block_num) this->vlist_cache.erase(it); } //this->vlist_cache.erase(_block_num); + unsigned i; + for(i = 0; i < 2000; ++i) + { + if (longlist[i].key == (int)_key) break; + } + if (i < 2000) // value is in cache + { + longlist[i].free(); + } unsigned store = _block_num, next; fseek(this->valfp, Address(store), SEEK_SET); @@ -379,19 +391,34 @@ VList::AddIntoCache(unsigned _key, char*& _str, unsigned _len) { // cout << "vlist start" << endl; unsigned node = _key % 2000; - while (longlist[node].key != -1) + unsigned i = node; + while (longlist[i].key != -1 && i != node - 1) { - ++node; - if (node > 1999) node %= 2000; + ++i; + if (i > 1999) i -= 2000; } - longlist[node].key = _key; - longlist[node]._str = new char [_len]; - memcpy(longlist[node]._str, _str, _len); - longlist[node]._len = _len; + if (longlist[i].key != -1) return; + longlist[i].key = _key; + longlist[i]._str = new char [_len]; + if (longlist[i]._str == NULL) + { + cout << "Vlist::AddIntoCache fail" << endl; + longlist[i].free(); + return; + } + memcpy(longlist[i]._str, _str, _len); + longlist[i]._len = _len; // cout << "vlist finish" << endl; // cout << "done vlist addintocache" << endl; } +void +VList::release_cache() +{ + for(int i = 0; i < 2000; i++) + longlist[i].free(); +} + VList::~VList() { //clear the cache @@ -400,8 +427,6 @@ VList::~VList() delete[] it->second; } this->vlist_cache.clear(); - /*for(int i = 0; i < 2000; i++) - delete [] longlist[i]._str;*/ //write the info back fseek(this->valfp, 0, SEEK_SET); fwrite(&cur_block_num, sizeof(unsigned), 1, valfp);//write current blocks num diff --git a/Util/VList.h b/Util/VList.h index 3d4d513..bb64f38 100644 --- a/Util/VList.h +++ b/Util/VList.h @@ -96,9 +96,10 @@ public: VList(std::string& _filepath, std::string& _mode, unsigned long long _buffer_size);//create a fixed-size file or open an existence bool readValue(unsigned _block_num, char*& _str, unsigned& _len, unsigned _key); unsigned writeValue(char* _str, unsigned _len); - bool removeValue(unsigned _block_num); + bool removeValue(unsigned _block_num, unsigned _key); ~VList(); + void release_cache(); static bool isLongList(unsigned _len); static bool listNeedDelete(unsigned _len); void AddIntoCache(unsigned _key, char*& _str, unsigned _len);