Merge branch 'qinzongyue'

fix bug in vlist cache with insertions or deletions

Database/Database.cpp

@@ -717,7 +717,7 @@ Database::load()
 	//vstree_thread.join();
 #endif
 	//load cache of sub2values and obj2values
-	this->load_cache();
+	//this->load_cache();
 	
 	//warm up always as finishing build(), to utilize the system buffer
 	//this->warmUp();
@@ -867,36 +867,55 @@ void
 Database::build_CacheOfPre2values()
 {
 	cout << "now add cache of preID2values..." << endl;
+	priority_queue <KEY_SIZE_VALUE, vector<KEY_SIZE_VALUE>, CmpByMod<2000> > temp_queue;
 	while (!candidate_preID.empty())
 	{
-		//cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl;
-		this->kvstore->AddIntoPreCache(candidate_preID.top().key);
+		temp_queue.push(candidate_preID.top());
 		candidate_preID.pop();
 	}
+	while (!temp_queue.empty())
+	{
+		//cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl;
+		this->kvstore->AddIntoPreCache(temp_queue.top().key);
+		temp_queue.pop();
+	}
 }
 
 void
 Database::build_CacheOfObj2values()
 {
 	cout << "now add cache of objID2values..." << endl;
+	// sort key according to their mod by 2000
+	priority_queue <KEY_SIZE_VALUE, vector<KEY_SIZE_VALUE>, CmpByMod<2000> > temp_queue;
 	while (!important_objID.empty())
 	{
-		//cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl;
-		this->kvstore->AddIntoObjCache(important_objID.top().key);
+		temp_queue.push(important_objID.top());
 		important_objID.pop();
 	}
+	while (!temp_queue.empty())
+	{
+		//cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl;
+		this->kvstore->AddIntoObjCache(temp_queue.top().key);
+		temp_queue.pop();
+	}
 }
 
 void
 Database::build_CacheOfSub2values()
 {
 	cout << "now add cache of subID2values..." << endl;
+	priority_queue <KEY_SIZE_VALUE, vector<KEY_SIZE_VALUE>, CmpByMod<2000> > temp_queue;
 	while (!important_subID.empty())
 	{
-		//cout << "add key " << important_subID.top().key << " size: " << important_subID.top().size << endl;
-		this->kvstore->AddIntoSubCache(important_subID.top().key);
+		temp_queue.push(important_subID.top());
 		important_subID.pop();
 	}
+	while (!temp_queue.empty())
+	{
+		//cout << "add key " << important_objID.top().key << " size: " << important_objID.top().size << endl;
+		this->kvstore->AddIntoSubCache(temp_queue.top().key);
+		temp_queue.pop();
+	}
 }
 
 void

KVstore/IVTree/IVTree.cpp

@@ -601,6 +601,7 @@ IVTree::save()	//save the whole tree to disk
 #ifdef DEBUG_KVSTORE
 	printf("now to save tree!\n");
 #endif
+	this->value_list->release_cache();
 	if (TSM->writeTree(this->root))
 		return true;
 	else

KVstore/IVTree/node/IVLeafNode.cpp

@@ -157,7 +157,7 @@ IVLeafNode::setValue(VList* _vlist, int _index, char* _str, unsigned _len, bool
 		cout<<"this is a vlist in set()"<<endl;
 #endif
 		unsigned block_num = this->values[_index].getLen();
-		_vlist->removeValue(block_num);
+		_vlist->removeValue(block_num, this->keys[_index]);
 	}
 	else
 	{
@@ -247,7 +247,7 @@ IVLeafNode::subValue(VList* _vlist, int _index, bool ifdel)
 	if(this->values[_index].isBstrLongList())
 	{
 		unsigned block_num = this->values[_index].getLen();
-		_vlist->removeValue(block_num);
+		_vlist->removeValue(block_num, this->keys[_index]);
 	}
 	else
 	{

Util/ClassForVlistCache.h

@@ -26,12 +26,25 @@ public:
 		_len = 0;
 		_str = NULL;
 	}
-	~Longlist_inMem()
+	void free()
 	{
 		if(_str != NULL)
 		{
 			delete [] _str;
 			_str = NULL;
 		}
+		key = -1;
+		_len = 0;
+	}
+};
+
+
+template <unsigned mod>
+class CmpByMod
+{
+public:
+	bool operator () (const KEY_SIZE_VALUE& a, const KEY_SIZE_VALUE& b)
+	{
+		return (a.key % mod) > (b.key % mod);
 	}
 };

Util/VList.cpp

@@ -208,8 +208,8 @@ VList::readValue(unsigned _block_num, char*& _str, unsigned& _len, unsigned _key
 		unsigned node = _key % 2000, i;
 		for(i = node; longlist[i].key % 2000 <= node; ++i)
 		{
-			if (longlist[i].key == (int)_key || longlist[i].key == -1) break;
-			if (i > 1999) i %= 2000;
+			if (longlist[i].key == (int)_key || longlist[i].key == -1 || i == node - 1) break;
+			if (i > 1999) i -= 2000;
 		}
 		if (longlist[i].key == (int)_key) // value is in cache
 		{
@@ -217,9 +217,12 @@ VList::readValue(unsigned _block_num, char*& _str, unsigned& _len, unsigned _key
 		//	accessOfCache++;		
 			_len = longlist[i]._len;
 			_str = new char[_len];
-			memcpy(_str, longlist[i]._str, _len);
-		//	cout << _len << endl;
-			return true;
+			if (_str != NULL)
+			{
+				memcpy(_str, longlist[i]._str, _len);
+			//	cout << _len << endl;
+				return true;
+			}
 		}
 
 	//if not in cache, read from disk(add a random seek time), the pointer should be clear in upper layer
@@ -271,7 +274,7 @@ VList::writeValue(char* _str, unsigned _len)
 }
 
 bool 
-VList::removeValue(unsigned _block_num)
+VList::removeValue(unsigned _block_num, unsigned _key)
 {
 	CACHE_ITERATOR it = this->vlist_cache.find(_block_num);
 	if(it != this->vlist_cache.end())
@@ -281,6 +284,15 @@ VList::removeValue(unsigned _block_num)
 		this->vlist_cache.erase(it);
 	}
 	//this->vlist_cache.erase(_block_num);
+	unsigned i;
+	for(i = 0; i < 2000; ++i)
+	{
+			if (longlist[i].key == (int)_key) break;
+	}
+	if (i < 2000) // value is in cache
+	{
+		longlist[i].free();
+	}
 
 	unsigned store = _block_num, next;
 	fseek(this->valfp, Address(store), SEEK_SET);
@@ -379,19 +391,34 @@ VList::AddIntoCache(unsigned _key, char*& _str, unsigned _len)
 {
 //	cout << "vlist start" << endl;
 	unsigned node = _key % 2000;
-	while (longlist[node].key != -1)
+	unsigned i = node;
+	while (longlist[i].key != -1 && i != node - 1)
 	{
-		++node;
-		if (node > 1999) node %= 2000;
+		++i;
+		if (i > 1999) i -= 2000;
 	}
-	longlist[node].key = _key;
-	longlist[node]._str = new char [_len];
-	memcpy(longlist[node]._str, _str, _len);
-	longlist[node]._len = _len;
+	if (longlist[i].key != -1) return;
+	longlist[i].key = _key;
+	longlist[i]._str = new char [_len];
+	if (longlist[i]._str == NULL)
+	{
+		cout << "Vlist::AddIntoCache fail" << endl;
+		longlist[i].free();
+		return;
+	}
+	memcpy(longlist[i]._str, _str, _len);
+	longlist[i]._len = _len;
 //	cout << "vlist finish" << endl;
 //	cout << "done vlist addintocache" << endl;
 }
 
+void
+VList::release_cache()
+{
+	for(int i = 0; i < 2000; i++)
+	longlist[i].free();	
+}
+
 VList::~VList()
 {
 	//clear the cache
@@ -400,8 +427,6 @@ VList::~VList()
 		delete[] it->second;
 	}
 	this->vlist_cache.clear();
-	/*for(int i = 0; i < 2000; i++)
-			delete [] longlist[i]._str;*/
 	//write the info back
 	fseek(this->valfp, 0, SEEK_SET);
 	fwrite(&cur_block_num, sizeof(unsigned), 1, valfp);//write current blocks num

Util/VList.h

@@ -96,9 +96,10 @@ public:
 	VList(std::string& _filepath, std::string& _mode, unsigned long long _buffer_size);//create a fixed-size file or open an existence
 	bool readValue(unsigned _block_num, char*& _str, unsigned& _len, unsigned _key);
 	unsigned writeValue(char* _str, unsigned _len);
-	bool removeValue(unsigned _block_num);
+	bool removeValue(unsigned _block_num, unsigned _key);
 	~VList();
 
+	void release_cache();
 	static bool isLongList(unsigned _len);
 	static bool listNeedDelete(unsigned _len);
 	void AddIntoCache(unsigned _key, char*& _str, unsigned _len);