opensci
/
gStore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

implement the new version of BTree, and replace the old one. 

author: zengli
This commit is contained in:
Caesar11 2015-09-25 07:05:59 +00:00
parent c8dfd99d00
commit 50d6c495e4
27 changed files with 3502 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
Database/Database.cpp
View File

@ -6,7 +6,6 @@
*/ */
#include "Database.h" #include "Database.h"
Database::Database(std::string _name){ Database::Database(std::string _name){
this->name = _name; this->name = _name;
std::string store_path = this->name; std::string store_path = this->name;
@ -221,7 +220,6 @@ bool Database::remove(const string& _rdf_file)
bool Database::build(const string& _rdf_file) bool Database::build(const string& _rdf_file)
{ {
long tv_build_begin = util::get_cur_time(); long tv_build_begin = util::get_cur_time();
bool flag = true;
std::string store_path = this->name; std::string store_path = this->name;
util::create_dir(store_path); util::create_dir(store_path);
@ -234,14 +232,8 @@ bool Database::build(const string& _rdf_file)
cout << "begin encode RDF from : " << _rdf_file << " ..." << endl; cout << "begin encode RDF from : " << _rdf_file << " ..." << endl;
// to be switched to new encodeRDF method. // to be switched to new encodeRDF method.
// flag = this->encodeRDF(_rdf_file); // this->encodeRDF(_rdf_file);
flag = this->encodeRDF_new(_rdf_file); this->encodeRDF_new(_rdf_file);
if (!flag)
{
std::cout << "encode RDF failed."<< std::endl;
return false;
}
cout << "finish encode." << endl; cout << "finish encode." << endl;
std::string _entry_file = this->getSignatureBFile(); std::string _entry_file = this->getSignatureBFile();
(this->kvstore)->open(); (this->kvstore)->open();
@ -481,27 +473,27 @@ bool Database::encodeRDF(const string _rdf_file)
bool Database::encodeRDF_new(const string _rdf_file) bool Database::encodeRDF_new(const string _rdf_file)
{ {
Database::log("In encodeRDF_new"); Database::log("In encodeRDF_new");
bool flag = true;
int ** _p_id_tuples = NULL; int ** _p_id_tuples = NULL;
int _id_tuples_max = 0; int _id_tuples_max = 0;
/* map sub2id, pre2id, entity/literal in obj2id, store in kvstore, encode RDF data into signature */ /* map sub2id, pre2id, entity/literal in obj2id, store in kvstore, encode RDF data into signature */
flag = this->sub2id_pre2id_obj2id_RDFintoSignature(_rdf_file, _p_id_tuples, _id_tuples_max); this->sub2id_pre2id_obj2id_RDFintoSignature(_rdf_file, _p_id_tuples, _id_tuples_max);
if (!flag) return false;
/* map subid 2 objid_list & /* map subid 2 objid_list &
* subIDpreID 2 objid_list & * subIDpreID 2 objid_list &
* subID 2 <preIDobjID>_list */ * subID 2 <preIDobjID>_list */
flag = this->s2o_sp2o_s2po(_p_id_tuples, _id_tuples_max); this->s2o_sp2o_s2po(_p_id_tuples, _id_tuples_max);
if (!flag) return false;
/* map objid 2 subid_list & /* map objid 2 subid_list &
* objIDpreID 2 subid_list & * objIDpreID 2 subid_list &
* objID 2 <preIDsubID>_list */ * objID 2 <preIDsubID>_list */
flag = this->o2s_op2s_o2ps(_p_id_tuples, _id_tuples_max); this->o2s_op2s_o2ps(_p_id_tuples, _id_tuples_max);
if (!flag) return false;
flag = this->saveDBInfoFile(); bool flag = this->saveDBInfoFile();
if (!flag) return false; if (!flag)
{
return false;
}
Database::log("finish encodeRDF_new"); Database::log("finish encodeRDF_new");
@ -534,14 +526,14 @@ bool Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file, int
ifstream _fin(_rdf_file.c_str()); ifstream _fin(_rdf_file.c_str());
if(!_fin){ if(!_fin){
cerr << "sub2id&pre2id&obj2id: Fail to open : " << _rdf_file << endl; cerr << "sub2id&pre2id&obj2id: Fail to open : " << _rdf_file << endl;
return false; exit(0);
} }
std::string _six_tuples_file = this->getSixTuplesFile(); std::string _six_tuples_file = this->getSixTuplesFile();
std::ofstream _six_tuples_fout(_six_tuples_file.c_str()); std::ofstream _six_tuples_fout(_six_tuples_file.c_str());
if(! _six_tuples_fout){ if(! _six_tuples_fout){
cerr << "sub2id&pre2id&obj2id: Fail to open: " << _six_tuples_file << endl; cerr << "sub2id&pre2id&obj2id: Fail to open: " << _six_tuples_file << endl;
return false; exit(0);
} }
TripleWithObjType* triple_array = new TripleWithObjType[RDFParser::TRIPLE_NUM_PER_GROUP]; TripleWithObjType* triple_array = new TripleWithObjType[RDFParser::TRIPLE_NUM_PER_GROUP];

1
Database/Database.h
View File

@ -9,6 +9,7 @@
#define DATABASE_H_ #define DATABASE_H_
#include<iostream> #include<iostream>
#include<string.h> #include<string.h>
#include <stack>
using namespace std; using namespace std;
#include "../Query/IDList.h" #include "../Query/IDList.h"

68
KVstore/KVstore.cpp
View File

@ -553,7 +553,7 @@ string KVstore::getEntityByID(int _id){
} }
} }
string _ret = string(_tmp); string _ret = string(_tmp);
delete[] _tmp; //delete[] _tmp; DEBUG
return _ret; return _ret;
} }
@ -602,7 +602,7 @@ bool KVstore::open_id2predicate(const int _mode){
string KVstore::getPredicateByID(int _id){ string KVstore::getPredicateByID(int _id){
char* _tmp = NULL; char* _tmp = NULL;
int _len = 0; int _len = 0;
bool _get = this->getValueByKey(this->id2predicate, (char*)&_id, sizeof(int), _tmp, _len); bool _get = this->getValueByKey(this->id2predicate, (const char*)&_id, sizeof(int), _tmp, _len);
{ {
if(!_get) if(!_get)
{ {
@ -610,7 +610,7 @@ string KVstore::getPredicateByID(int _id){
} }
} }
string _ret = string(_tmp); string _ret = string(_tmp);
delete[] _tmp; //delete[] _tmp;
return _ret; return _ret;
} }
@ -667,7 +667,7 @@ string KVstore::getLiteralByID(int _id){
} }
} }
string _ret = string(_tmp); string _ret = string(_tmp);
delete[] _tmp; // delete[] _tmp;
return _ret; return _ret;
} }
@ -707,7 +707,7 @@ bool KVstore::getobjIDlistBysubID(int _subid, int*& _objidlist, int& _list_len){
_objidlist = new int[_list_len]; _objidlist = new int[_list_len];
memcpy((char*)_objidlist, _tmp, sizeof(int)*_list_len); memcpy((char*)_objidlist, _tmp, sizeof(int)*_list_len);
} }
delete[] _tmp; // delete[] _tmp;
return true; return true;
} }
@ -739,7 +739,7 @@ bool KVstore::getsubIDlistByobjID(int _objid, int*& _subidlist, int& _list_len){
_subidlist = new int[_list_len]; _subidlist = new int[_list_len];
memcpy((char*)_subidlist, _tmp, sizeof(int)*_list_len); memcpy((char*)_subidlist, _tmp, sizeof(int)*_list_len);
} }
delete[] _tmp; //delete[] _tmp;
return true; return true;
} }
@ -776,7 +776,7 @@ bool KVstore::getobjIDlistBysubIDpreID(int _subid, int _preid, int*& _objidlist,
_objidlist = new int[_list_len]; _objidlist = new int[_list_len];
memcpy((char*)_objidlist, _tmp, sizeof(int)*_list_len); memcpy((char*)_objidlist, _tmp, sizeof(int)*_list_len);
} }
delete[] _tmp; //delete[] _tmp;
return true; return true;
} }
@ -825,7 +825,7 @@ bool KVstore::getsubIDlistByobjIDpreID(int _objid, int _preid, int*& _subidlist,
_subidlist = new int[_list_len]; _subidlist = new int[_list_len];
memcpy((char*)_subidlist, _tmp, sizeof(int)*_list_len); memcpy((char*)_subidlist, _tmp, sizeof(int)*_list_len);
} }
delete[] _tmp; //delete[] _tmp;
return true; return true;
} }
@ -865,7 +865,7 @@ bool KVstore::getpreIDobjIDlistBysubID(int _subid, int*& _preid_objidlist, int&
_preid_objidlist = new int[_list_len]; _preid_objidlist = new int[_list_len];
memcpy((char*)_preid_objidlist, _tmp, sizeof(int)*_list_len); memcpy((char*)_preid_objidlist, _tmp, sizeof(int)*_list_len);
} }
delete[] _tmp; //delete[] _tmp;
return true; return true;
} }
@ -898,7 +898,7 @@ bool KVstore::getpreIDsubIDlistByobjID(int _objid, int*& _preid_subidlist, int&
_preid_subidlist = new int[_list_len]; _preid_subidlist = new int[_list_len];
memcpy((char*)_preid_subidlist, _tmp, sizeof(int)*_list_len); memcpy((char*)_preid_subidlist, _tmp, sizeof(int)*_list_len);
} }
delete[] _tmp; //delete[] _tmp;
return true; return true;
} }
@ -909,7 +909,7 @@ bool KVstore::setpreIDsubIDlistByobjID(int _objid, const int* _preid_subidlist,
} }
/* set the store_path as the root dir of this KVstore /* set the store_path as the root dir of this KVstore
* initial all Btree pointer as NULL * initial all Tree pointer as NULL
* */ * */
KVstore::KVstore(const string _store_path){ KVstore::KVstore(const string _store_path){
this->store_path = _store_path; this->store_path = _store_path;
@ -938,7 +938,8 @@ KVstore::KVstore(const string _store_path){
* before destruction * before destruction
* */ * */
KVstore::~KVstore(){ KVstore::~KVstore(){
this->release(); //this->release();
this->flush();
delete this->entity2id; delete this->entity2id;
delete this->id2entity; delete this->id2entity;
@ -962,7 +963,7 @@ KVstore::~KVstore(){
/* /*
* just flush all modified part into disk * just flush all modified part into disk
* will not release any memory at all * will not release any memory at all
* any Btree pointer that is null or * any Tree pointer that is null or
* has not been modified will do nothing * has not been modified will do nothing
* */ * */
void KVstore::flush(){ void KVstore::flush(){
@ -985,8 +986,8 @@ void KVstore::flush(){
this->flush(this->objID2preIDsubIDlist); this->flush(this->objID2preIDsubIDlist);
} }
/* Release all the memory used in this KVstore, /* Release all the memory used in this KVstore,
* following an flush() for each Btree pointer * following an flush() for each Tree pointer
* any Btree pointer that is null or * any Tree pointer that is null or
* has not been modified will do nothing * has not been modified will do nothing
* */ * */
void KVstore::release(){ void KVstore::release(){
@ -1035,23 +1036,26 @@ void KVstore::open()
/* /*
* private methods: * private methods:
*/ */
void KVstore::flush(Btree* _p_btree){ void KVstore::flush(Tree* _p_btree){
if(_p_btree != NULL) if(_p_btree != NULL)
{ {
_p_btree->flush(); _p_btree->save();
} }
} }
void KVstore::release(Btree* _p_btree){
void KVstore::release(Tree* _p_btree){
/*
if(_p_btree != NULL) if(_p_btree != NULL)
{ {
_p_btree->release(); _p_btree->save();
} }
*/
} }
/* Open a btree according the mode */ /* Open a btree according the mode */
/* CREATE_MODE: build a new btree and delete if exist */ /* CREATE_MODE: build a new btree and delete if exist */
/* READ_WRITE_MODE: open a btree, btree must exist */ /* READ_WRITE_MODE: open a btree, btree must exist */
bool KVstore::open(Btree* & _p_btree, const string _tree_name, const int _mode){ bool KVstore::open(Tree* & _p_btree, const string _tree_name, const int _mode){
if(_p_btree != NULL) if(_p_btree != NULL)
{ {
return false; return false;
@ -1059,13 +1063,13 @@ bool KVstore::open(Btree* & _p_btree, const string _tree_name, const int _mode){
if(_mode == KVstore::CREATE_MODE) if(_mode == KVstore::CREATE_MODE)
{ {
_p_btree = new Btree(this->store_path, _tree_name, "w"); _p_btree = new Tree(this->store_path, _tree_name, "build");
return true; return true;
} }
else else
if(_mode == KVstore::READ_WRITE_MODE) if(_mode == KVstore::READ_WRITE_MODE)
{ {
_p_btree = new Btree(this->store_path, _tree_name, "rw"); _p_btree = new Tree(this->store_path, _tree_name, "open");
return true; return true;
} }
else else
@ -1076,26 +1080,32 @@ bool KVstore::open(Btree* & _p_btree, const string _tree_name, const int _mode){
return false; return false;
} }
bool KVstore::setValueByKey(Btree* _p_btree, const char* _key, int _klen, const char* _val, int _vlen){ //DEBUG:not achieve multiple-type functions, may have to organize in Bstr, or add functions in btree
bool KVstore::setValueByKey(Tree* _p_btree, const char* _key, int _klen, const char* _val, int _vlen){
return _p_btree->insert(_key, _klen, _val, _vlen); return _p_btree->insert(_key, _klen, _val, _vlen);
} }
bool KVstore::getValueByKey(Btree* _p_btree, const char* _key, int _klen, char*& _val, int& _vlen){ bool KVstore::getValueByKey(Tree* _p_btree, const char* _key, int _klen, char*& _val, int& _vlen){
return _p_btree->search(_key, _klen, _val, _vlen); return _p_btree->search(_key, _klen, _val, _vlen);
} }
int KVstore::getIDByStr(Btree* _p_btree, const char* _key, int _klen) int KVstore::getIDByStr(Tree* _p_btree, const char* _key, int _klen)
{
char* val = NULL;
int vlen = 0;
bool ret = _p_btree->search(_key, _klen, val, vlen);
if(!ret) //QUERY: if need to check vlen?
{ {
bool _ret = _p_btree->search(_key, _klen);
if(!_ret){
return -1; return -1;
} }
/* int is stored in str /* int is stored in str
* forcely change str into int* and, get the int value with '*' */ * forcely change str into int* and, get the int value with '*' */
return *( (int*)( (_p_btree->getValueTransfer())->str ) ); //return *( (int*)( (_p_btree->getValueTransfer())->str ) );
return *((int*)val);
} }
bool KVstore::removeKey(Btree* _p_btree, const char* _key, int _klen) bool KVstore::removeKey(Tree* _p_btree, const char* _key, int _klen)
{ {
return _p_btree->remove(_key, _klen); return _p_btree->remove(_key, _klen);
} }

41
KVstore/KVstore.h
View File

@ -11,8 +11,9 @@
#include <string.h> #include <string.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <dirent.h> #include <dirent.h>
#include"Btree.h" #include "tree/Tree.h"
using namespace std; using namespace std;
class KVstore{ class KVstore{
public: public:
static const bool debug_mode = false; static const bool debug_mode = false;
@ -126,50 +127,50 @@ private:
* map entity to its id, and id to the entity * map entity to its id, and id to the entity
* s_entity2id is relative store file name * s_entity2id is relative store file name
*/ */
Btree* entity2id; Tree* entity2id;
Btree* id2entity; Tree* id2entity;
static string s_entity2id; static string s_entity2id;
static string s_id2entity; static string s_id2entity;
Btree* predicate2id; Tree* predicate2id;
Btree* id2predicate; Tree* id2predicate;
static string s_predicate2id; static string s_predicate2id;
static string s_id2predicate; static string s_id2predicate;
Btree* literal2id; Tree* literal2id;
Btree* id2literal; Tree* id2literal;
static string s_literal2id; static string s_literal2id;
static string s_id2literal; static string s_id2literal;
Btree* subID2objIDlist; Tree* subID2objIDlist;
Btree* objID2subIDlist; Tree* objID2subIDlist;
static string s_sID2oIDlist; static string s_sID2oIDlist;
static string s_oID2sIDlist; static string s_oID2sIDlist;
/* lack exist in update tuple */ /* lack exist in update tuple */
Btree* subIDpreID2objIDlist; Tree* subIDpreID2objIDlist;
Btree* objIDpreID2subIDlist; Tree* objIDpreID2subIDlist;
static string s_sIDpID2oIDlist; static string s_sIDpID2oIDlist;
static string s_oIDpID2sIDlist; static string s_oIDpID2sIDlist;
Btree* subID2preIDobjIDlist; Tree* subID2preIDobjIDlist;
Btree* objID2preIDsubIDlist; Tree* objID2preIDsubIDlist;
static string s_sID2pIDoIDlist; static string s_sID2pIDoIDlist;
static string s_oID2pIDsIDlist; static string s_oID2pIDsIDlist;
void flush(Btree* _p_btree); void flush(Tree* _p_btree);
void release(Btree* _p_btree); void release(Tree* _p_btree);
bool setValueByKey(Btree* _p_btree, const char* _key, int _klen, const char* _val, int _vlen); bool setValueByKey(Tree* _p_btree, const char* _key, int _klen, const char* _val, int _vlen);
bool getValueByKey(Btree* _p_btree, const char* _key, int _klen, char*& _val, int& _vlen); bool getValueByKey(Tree* _p_btree, const char* _key, int _klen, char*& _val, int& _vlen);
int getIDByStr(Btree* _p_btree, const char* _key, int _klen); int getIDByStr(Tree* _p_btree, const char* _key, int _klen);
bool removeKey(Btree* _p_btree, const char* _key, int _klen); bool removeKey(Tree* _p_btree, const char* _key, int _klen);
/* Open a btree according the mode */ /* Open a btree according the mode */
/* CREATE_MODE: build a new btree and delete if exist */ /* CREATE_MODE: build a new btree and delete if exist */
/* READ_WRITE_MODE: open a btree, btree must exist */ /* READ_WRITE_MODE: open a btree, btree must exist */
bool open(Btree* & _p_btree, const string _tree_name, const int _mode); bool open(Tree* & _p_btree, const string _tree_name, const int _mode);
}; };

31
KVstore/Makefile Normal file
View File

@ -0,0 +1,31 @@
CC = g++
CFLAGS = -Wall -Werror -c -g
objdir = ../objs/
obj = $(objdir)Tree.o $(objdir)Storage.o $(objdir)Node.o $(objdir)IntlNode.o \
$(objdir)LeafNode.o $(objdir)TBstr.o $(objdir)Util.o $(objdir)Heap.o $(objdir)Hash.o $(objdir)RangeValue.o
all: $(obj)
$(objdir)Tree.o: tree/Tree.cpp
$(CC) $(CFLAGS) tree/Tree.cpp -o $(objdir)Tree.o
$(objdir)Storage.o: storage/Storage.cpp
$(CC) $(CFLAGS) storage/Storage.cpp -o $(objdir)Storage.o
$(objdir)Node.o: node/Node.cpp
$(CC) $(CFLAGS) node/Node.cpp -o $(objdir)Node.o
$(objdir)IntlNode.o: node/IntlNode.cpp
$(CC) $(CFLAGS) node/IntlNode.cpp -o $(objdir)IntlNode.o
$(objdir)LeafNode.o: node/LeafNode.cpp
$(CC) $(CFLAGS) node/LeafNode.cpp -o $(objdir)LeafNode.o
$(objdir)TBstr.o: bstr/TBstr.cpp
$(CC) $(CFLAGS) bstr/TBstr.cpp -o $(objdir)TBstr.o
$(objdir)Util.o: util/Util.cpp
$(CC) $(CFLAGS) util/Util.cpp -o $(objdir)Util.o
$(objdir)Hash.o: hash/Hash.cpp
$(CC) $(CFLAGS) hash/Hash.cpp -o $(objdir)Hash.o
$(objdir)Heap.o: heap/Heap.cpp
$(CC) $(CFLAGS) heap/Heap.cpp -o $(objdir)Heap.o
$(objdir)RangeValue.o: rangevalue/RangeValue.cpp
$(CC) $(CFLAGS) rangevalue/RangeValue.cpp -o $(objdir)RangeValue.o
.PHONY: clean
clean:
#nothing to do now

159
KVstore/bstr/TBstr.cpp Normal file
View File

@ -0,0 +1,159 @@
/*=============================================================================
# Filename: TBstr.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:35
# Description: achieve functions in TBstr.h
=============================================================================*/
#include "TBstr.h"
using namespace std;
TBstr::TBstr()
{
this->length = 0;
this->str = NULL;
}
TBstr::TBstr(char* _str, unsigned _len)
{
this->length = _len;
this->str = _str;
}
bool
TBstr::operator > (const TBstr& _bstr)
{
int res = Util::compare(this->str, this->length, _bstr.str, _bstr.length);
if(res == 1)
return true;
else
return false;
}
bool
TBstr::operator < (const TBstr& _bstr)
{
int res = Util::compare(this->str, this->length, _bstr.str, _bstr.length);
if(res == -1)
return true;
else
return false;
}
bool
TBstr::operator == (const TBstr& _bstr)
{
int res = Util::compare(this->str, this->length, _bstr.str, _bstr.length);
if(res == 0)
return true;
else
return false;
}
bool
TBstr::operator <= (const TBstr& _bstr)
{
int res = Util::compare(this->str, this->length, _bstr.str, _bstr.length);
if(res <= 0)
return true;
else
return false;
}
bool
TBstr::operator >= (const TBstr& _bstr)
{
int res = Util::compare(this->str, this->length, _bstr.str, _bstr.length);
if(res >= 0)
return true;
else
return false;
}
bool
TBstr::operator != (const TBstr& _bstr)
{
int res = Util::compare(this->str, this->length, _bstr.str, _bstr.length);
if(res != 0)
return true;
else
return false;
}
unsigned
TBstr::getLen() const
{
return length;
}
void
TBstr::setLen(unsigned _len)
{
this->length = _len;
}
char*
TBstr::getStr() const
{
return str;
}
void
TBstr::setStr(char* _str)
{
this->str = _str;
}
void
TBstr::copy(const TBstr* _bp)
{
this->length = _bp->getLen();
this->str = (char*)malloc(this->length);
memcpy(this->str, _bp->getStr(), this->length);
}
void
TBstr::clear()
{
this->str = NULL;
this->length = 0;
}
void
TBstr::release()
{
free(this->str); //ok to be null, do nothing
clear();
}
TBstr::~TBstr()
{ //avoid mutiple delete
release();
}
void
TBstr::print(string s) const
{
#ifdef DEBUG
Util::showtime();
fputs("Class TBstr\n", Util::logsfp);
fputs("Message: ", Util::logsfp);
fputs(s.c_str(), Util::logsfp);
fputs("\n", Util::logsfp);
if(s == "BSTR")
{ //total information, providing accurate debugging
fprintf(Util::logsfp, "length: %u\t the string is:\n", this->length);
unsigned i;
for(i = 0; i < this->length; ++i)
fputc(this->str[i], Util::logsfp);
fputs("\n", Util::logsfp);
}
else if(s == "bstr")
{ //only length information, needed when string is very long
fprintf(Util::logsfp, "length: %u\n", this->length);
}
else;
#endif
}

40
KVstore/bstr/TBstr.h Normal file
View File

@ -0,0 +1,40 @@
/*=============================================================================
# Filename: TBstr.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:33
# Description: class declaration for TBstr(used to store arbitary string)
=============================================================================*/
#ifndef _T_BSTR_H
#define _T_BSTR_H
#include "../util/Util.h"
class TBstr
{
private:
char* str; //pointers consume 8 byte in 64-bit system
unsigned length;
public:
TBstr();
TBstr(char* _str, unsigned _len);
bool operator > (const TBstr& _bstr);
bool operator < (const TBstr& _bstr);
bool operator == (const TBstr& _bstr);
bool operator <= (const TBstr& _bstr);
bool operator >= (const TBstr& _bstr);
bool operator != (const TBstr& _bstr);
unsigned getLen() const;
void setLen(unsigned _len);
char* getStr() const;
void setStr(char* _str); //reuse a TBstr
void release(); //release memory
void clear(); //set str/length to 0
void copy(const TBstr* _bp);
~TBstr();
void print(std::string s) const; //DEBUG
};
#endif

21
KVstore/hash/Hash.cpp Normal file
View File

@ -0,0 +1,21 @@
/*=============================================================================
# Filename: Hash.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:36
# Description: achieve functions in Hash.h
=============================================================================*/
#include "Hash.h"
using namespace std;
Hash::Hash()
{
}
Hash::Hash(unsigned _size)
{
}

28
KVstore/hash/Hash.h Normal file
View File

@ -0,0 +1,28 @@
/*=============================================================================
# Filename: Hash.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:36
# Description: to utilize the search for Node* NOT USED YET
=============================================================================*/
#ifndef _HASH_H
#define _HASH_H
#include "../util/Util.h"
#include "../node/Node.h"
class Hash
{
private:
Node* np;
unsigned position;
unsigned next;
unsigned size;
public:
Hash();
Hash(unsigned _size);
};
#endif

186
KVstore/heap/Heap.cpp Normal file
View File

@ -0,0 +1,186 @@
/*=============================================================================
# Filename: Heap.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:37
# Description: achieve functions in Heap.h
=============================================================================*/
#include "Heap.h"
using namespace std;
Heap::Heap()
{
this->length = this->size = 0;
this->heap = NULL;
this->hash = NULL;
}
Heap::Heap(unsigned _size)
{
this->length = 0;
this->size = _size;
this->heap = (Node**)malloc(this->size * sizeof(Node*)); //not use 4 or 8
if(this->heap == NULL)
{
this->print("error in Heap: Allocation fail!");
exit(1);
}
/*
this->npmap = (Map*)malloc(this->size * sizeof(struct Map));
if(this->npmap == NULL)
{
this->print("error in Heap: Allocation fail!");
exit(1);
}
*/
}
Node*
Heap::getTop() const
{
if(this->length > 0)
return this->heap[0];
else
return NULL;
}
unsigned
Heap::getLen() const
{
return this->length;
}
unsigned
Heap::getSize() const
{
return this->size;
}
bool
Heap::isEmpty() const
{
return this->length == 0;
}
bool
Heap::insert(Node* _np)
{
if(this->length == this->size) //when full, reallocate
{
this->heap = (Node**)realloc(this->heap, 2 * this->size * sizeof(Node*));
if(this->heap == NULL)
{
print("error in isert: Reallocation fail!");
return false;
}
/*
this->npmap = (struct Map*)realloc(this->npmap, 2 * this->size * sizeof(struct Map));
if(this->npmap == NULL)
{
print("error in insert: Reallocation fail!");
return false;
}
*/
this->size = 2 * this->size;
}
unsigned i = this->length, j;
while(i != 0)
{
j = (i-1)/2;
if(_np->getRank() >= this->heap[j]->getRank())
break;
heap[i] = heap[j];
//this->npmap[k].pos = i; //adjust the position
i = j;
}
this->heap[i] = _np;
this->length++;
return true;
}
bool
Heap::remove()
{
if(this->length == 0)
{
print("error in remove: remove from empty heap!");
return false;
}
//Node* tp = this->heap[0];
this->length--;
if(this->length == 0)
return true;
Node* xp = this->heap[this->length];
unsigned i = 0, j = 1;
while(j < this->length)
{
if(j < this->length-1 && this->heap[j]->getRank() > this->heap[j+1]->getRank())
j++;
if(xp->getRank() <= this->heap[j]->getRank())
break;
this->heap[i] = this->heap[j];
i = j;
j = 2 * i + 1;
}
this->heap[i] = xp;
return true;
}
bool
Heap::modify(Node* _np, bool _flag) //control direction
{
//search and adjust
unsigned i, j;
for(i = 0; i < this->length; ++i)
if(this->heap[i] == _np)
break;
if(_flag == true) //move up
{
while(i != 0)
{
j = (i-1)/2;
if(_np->getRank() < heap[j]->getRank())
{
heap[i] = heap[j];
heap[j] = _np;
i = j;
}
else
break;
}
}
else //move down
{
j = 2 * i + 1;
while(j < this->length)
{
if(j < this->length - 1 && heap[j]->getRank() > heap[j+1]->getRank())
j++;
if(heap[j]->getRank() < _np->getRank())
{
heap[i] = heap[j];
heap[j] = _np;
i = j;
}
else
break;
}
}
return true;
}
Heap::~Heap()
{
delete[] this->heap;
this->heap = NULL;
this->length = this->size = 0;
}
void
Heap::print(string s)
{
#ifdef DEBUG
#endif
}

44
KVstore/heap/Heap.h Normal file
View File

@ -0,0 +1,44 @@
/*=============================================================================
# Filename: Heap.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:37
# Description: set and deal of Node*s in memory
=============================================================================*/
#ifndef _HEAP_H
#define _HEAP_H
#include "../util/Util.h"
#include "../node/Node.h"
#include "../hash/Hash.h"
/* add, sub, modify: all can be done within O(logn) using adjust-function */
//QUERY: when modified, finding right position consumes O(n). How about keeping smallest?
//(add O(1), sub O(2n), modify O(n)
//TODO: to solve this probem, use another hash: (pointer, pos), to find the right position of
//given p in O(lgn) time
class Heap
{
private:
Node** heap; //dynamic array
Hash* hash; //vary according to heap
unsigned length; //valid elements num
unsigned size; //max-size of heap
public:
Heap();
Heap(unsigned _size);
Node* getTop() const; //return the top element
unsigned getLen() const;
unsigned getSize() const;
bool isEmpty() const;
bool insert(Node* _np); //insert and adjust
bool remove(); //remove top and adjust
bool modify(Node* _np, bool _flag); //searech modified element and adjust
~Heap();
void print(std::string s); //DEBUG
};
#endif

293
KVstore/node/IntlNode.cpp Normal file
View File

@ -0,0 +1,293 @@
/*=============================================================================
# Filename: IntlNode.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:40
# Description: achieve functions in IntlNode.h
=============================================================================*/
#include "IntlNode.h"
using namespace std;
/*
void
IntlNode::AllocChilds()
{
childs = (Node**)malloc(sizeof(Node*) * MAX_CHILD_NUM);
}
*/
IntlNode::IntlNode()
{
memset(childs, 0, sizeof(Node*) * MAX_CHILD_NUM);
//this->AllocChilds();
}
IntlNode::IntlNode(bool isVirtual) //call father-class's constructor automaticlly
{
memset(childs, 0, sizeof(Node*) * MAX_CHILD_NUM);
//this->AllocChilds();
}
/*
IntlNode::IntlNode(Storage* TSM) //QUERY
{
TSM->readNode(this, Storage::OVER);
}
*/
void
IntlNode::Virtual()
{
//this->FreeKeys();
this->release();
this->delMem();
}
void
IntlNode::Normal()
{
this->AllocKeys();
this->setMem();
}
Node*
IntlNode::getChild(int _index) const
{
int num = this->getNum();
if(_index < 0 || _index > num) //num keys, num+1 childs
{
//print(string("error in getChild: Invalid index ") + Util::int2string(_index));
return NULL;
}
else
return childs[_index];
}
bool
IntlNode::setChild(Node* _child, int _index)
{
int num = this->getNum();
if(_index < 0 || _index > num)
{
print(string("error in setChild: Invalid index ") + Util::int2string(_index));
return false;
}
this->childs[_index] = _child;
return true;
}
bool
IntlNode::addChild(Node* _child, int _index)
{
int num = this->getNum();
if(_index < 0 || _index > num+1)
{
print(string("error in addChild: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
for(i = num; i >= _index; --i) //DEBUG: right bounder!!!
childs[i+1] = childs[i];
childs[_index] = _child;
return true;
}
bool
IntlNode::subChild(int _index)
{
int num = this->getNum();
if(_index < 0 || _index > num)
{
print(string("error in subchild: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
for(i = _index; i < num; ++i) //DEBUG: right bounder!!!
childs[i] = childs[i+1];
return true;
}
unsigned
IntlNode::getSize() const
{
unsigned sum = INTL_SIZE, num = this->getNum(), i;
for(i = 0; i < num; ++i)
sum += keys[i].getLen();
return sum;
}
Node*
IntlNode::split(Node* _father, int _index)
{
int num = this->getNum();
Node* p = new IntlNode; //right child
p->setHeight(this->getHeight());
int i, k;
for(i = MIN_CHILD_NUM, k = 0; i < num; ++i, ++k)
{
p->addKey(this->keys+i, k);
p->addChild(this->childs[i], k);
p->addNum();
}
p->addChild(this->childs[i], k);
const TBstr* tp = this->keys + MIN_KEY_NUM;
this->setNum(MIN_KEY_NUM);
_father->addKey(tp, _index);
_father->addChild(p, _index+1); //DEBUG(check the index)
_father->addNum();
_father->setDirty();
p->setDirty();
this->setDirty();
return p;
}
Node*
IntlNode::coalesce(Node* _father, int _index)
{
//int num = this->getNum();
int i, j = _father->getNum(), k; //BETTER: unsigned?
Node* p;
int ccase = 0;
const TBstr* bstr;
if(_index < j) //the right neighbor
{
p = _father->getChild(_index+1);
k = p->getNum();
if((unsigned)k > MIN_KEY_NUM)
ccase = 2;
else //==MIN_KEY_NUM
ccase = 1;
}
if(_index > 0) //the left neighbor
{
Node* tp = _father->getChild(_index-1);
unsigned tk = tp->getNum();
if(ccase < 2)
{
if(ccase == 0)
ccase = 3;
if(tk > MIN_KEY_NUM)
ccase = 4;
}
if(ccase > 2)
{
p = tp;
k = tk;
}
}
switch(ccase)
{
case 1: //union right to this
this->addKey(_father->getKey(_index), this->getNum());
this->addNum();
for(i = 0; i < k; ++i)
{
this->addKey(p->getKey(i), this->getNum());
this->addChild(p->getChild(i), this->getNum());
this->addNum();
}
this->setChild(p->getChild(i), this->getNum());
_father->subKey(_index);
_father->subChild(_index+1);
_father->subNum();
p->setNum(0);
//delete p;
break;
case 2: //move one form right
this->addKey(_father->getKey(_index), this->getNum());
_father->setKey(p->getKey(0), _index);
p->subKey(0);
this->addChild(p->getChild(0), this->getNum()+1);
p->subChild(0);
this->addNum();
p->subNum();
break;
case 3: //union left to this
this->addKey(_father->getKey(_index-1), 0);
this->addNum();
for(i = k; i > 0; --i)
{
int t = i - 1;
this->addKey(p->getKey(t), 0);
this->addChild(p->getChild(i), 0);
this->addNum();
}
this->addChild(p->getChild(0), 0);
_father->subKey(_index-1);
_father->subChild(_index-1);
_father->subNum();
p->setNum(0);
//delete p;
break;
case 4: //move one from left
bstr = p->getKey(k-1);
p->subKey(k-1);
this->addKey(_father->getKey(_index-1), 0);
_father->setKey(bstr, _index-1);
this->addChild(p->getChild(k), 0);
p->subChild(k);
this->addNum();
p->subNum();
break;
default:
print("error in coalesce: Invalid case!");
//printf("error in coalesce: Invalid case!");
}
_father->setDirty();
p->setDirty();
this->setDirty();
if(ccase == 1 || ccase == 3)
return p;
else
return NULL;
}
void
IntlNode::release()
{
if(!this->inMem())
return;
unsigned num = this->getNum();
//delete[] keys; //this will release all!!!
for(unsigned i = num; i < MAX_KEY_NUM; ++i)
keys[i].clear();
delete[] keys;
}
IntlNode::~IntlNode()
{
release();
//free(childs);
}
void
IntlNode::print(string s)
{
#ifdef DEBUG
int num = this->getNum();
Util::showtime();
fputs("Class IntlNode\n", Util::logsfp);
fputs("Message: ", Util::logsfp);
fputs(s.c_str(), Util::logsfp);
fputs("\n", Util::logsfp);
if(s == "node" || s == "NODE")
{
fprintf(Util::logsfp, "store: %u\tnum: %u\tflag: %u\n", this->store, num, this->flag);
int i;
for(i = 0; i < num; ++i)
{
if(s == "node")
this->keys[i].print("bstr");
else
this->keys[i].print("BSTR");
}
}
else if(s == "check node")
{
//TODO(check node, if satisfy B+ definition)
}
else;
#endif
}

50
KVstore/node/IntlNode.h Normal file
View File

@ -0,0 +1,50 @@
/*=============================================================================
# Filename: IntlNode.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:40
# Description: the internal-node of a B+ tree
=============================================================================*/
#ifndef _INTL_NODE_H
#define _INTL_NODE_H
#include "../util/Util.h"
#include "Node.h"
class IntlNode: public Node
{
protected:
Node* childs[MAX_CHILD_NUM+1];
//Node** childs;
//void AllocChilds();
public:
IntlNode();
IntlNode(bool isVirtual);
//IntlNode(Storage* TSM);
void Virtual();
void Normal();
Node* getChild(int _index) const;
bool setChild(Node* _child, int _index);
bool addChild(Node* _child, int _index);
bool subChild(int _index);
unsigned getSize() const;
Node* split(Node* _father, int _index);
Node* coalesce(Node* _father, int _index);
void release();
~IntlNode();
void print(std::string s); //DEBUG
/*non-sense functions: polymorphic
Node* getPrev() const;
Node* getNext() const;
const TBstr* getValue(int _index) const;
bool setValue(const TBstr* _value, int _index);
bool addValue(const TBstr* _value, int _index);
bool subValue(int _index);
void setPrev(Node* _prev);
void setNext(Node* _next);
*/
};
#endif

376
KVstore/node/LeafNode.cpp Normal file
View File

@ -0,0 +1,376 @@
/*=============================================================================
# Filename: LeafNode.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:40
# Description: ahieve functions in LeafNode.h
=============================================================================*/
#include "LeafNode.h"
using namespace std;
void
LeafNode::AllocValues()
{
values = new TBstr[MAX_KEY_NUM];
}
/*
void
LeafNode::FreeValues()
{
delete[] values;
}
*/
LeafNode::LeafNode()
{
flag |= NF_IL; //leaf flag
prev = next = NULL;
AllocValues();
}
LeafNode::LeafNode(bool isVirtual)
{
flag |= NF_IL;
prev = next = NULL;
if(!isVirtual)
AllocValues();
}
/*
LeafNode::LeafNode(Storage* TSM)
{
AllocValues();
TSM->readNode(this, Storage::OVER);
}
*/
void
LeafNode::Virtual()
{
//this->FreeKeys();
//this->FreeValues();
this->release();
this->delMem();
}
void
LeafNode::Normal()
{
this->AllocKeys();
this->AllocValues();
this->setMem();
}
Node*
LeafNode::getPrev() const
{
return prev;
}
Node*
LeafNode::getNext() const
{
return next;
}
const TBstr*
LeafNode::getValue(int _index) const
{
int num = this->getNum();
if(_index < 0 || _index >= num)
{
//print(string("error in getValue: Invalid index ") + Util::int2string(_index));
return NULL;
}
else
return this->values + _index;
}
bool
LeafNode::setValue(const TBstr* _value, int _index, bool ifcopy)
{
int num = this->getNum();
if(_index < 0 || _index >= num)
{
print(string("error in setValue: Invalid index ") + Util::int2string(_index));
return false;
}
this->values[_index].release(); //NOTICE: only used in modify
if(ifcopy)
this->values[_index].copy(_value);
else
this->values[_index] = *_value;
return true;
}
bool
LeafNode::addValue(const TBstr* _value, int _index, bool ifcopy)
{
int num = this->getNum();
if(_index < 0 || _index > num)
{
print(string("error in addValue: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
for(i = num-1; i >= _index; --i)
this->values[i+1] = this->values[i];
if(ifcopy)
this->values[_index].copy(_value);
else
this->values[_index] = *_value;
return true;
}
bool
LeafNode::subValue(int _index, bool ifdel)
{
int num = this->getNum();
if(_index < 0 || _index >= num)
{
print(string("error in subValue: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
if(ifdel)
values[_index].release();
for(i = _index; i < num-1; ++i)
this->values[i] = this->values[i+1];
return true;
}
void
LeafNode::setPrev(Node* _prev)
{
this->prev = _prev;
}
void
LeafNode::setNext(Node* _next)
{
this->next = _next;
}
unsigned
LeafNode::getSize() const
{
unsigned sum = LEAF_SIZE, num = this->getNum(), i;
for(i = 0; i < num; ++i)
{
sum += keys[i].getLen();
sum += values[i].getLen();
}
return sum;
}
Node*
LeafNode::split(Node* _father, int _index)
{
int num = this->getNum();
Node* p = new LeafNode; //right child
p->setHeight(this->getHeight()); //NOTICE: assign height for new node
p->setNext(this->next);
this->setNext(p);
p->setPrev(this);
int i, k;
for(i = MIN_KEY_NUM, k = 0; i < num; ++i, ++k)
{
p->addKey(this->keys+i, k);
p->addValue(this->values+i, k);
p->addNum();
}
const TBstr* tp = this->keys + MIN_KEY_NUM;
this->setNum(MIN_KEY_NUM);
_father->addKey(tp, _index, true);
_father->addChild(p, _index+1); //DEBUG(check the index)
_father->addNum();
_father->setDirty();
p->setDirty();
this->setDirty();
return p;
}
Node*
LeafNode::coalesce(Node* _father, int _index)
{ //add a key or coalesce a neighbor to this
int i, j = _father->getNum(), k; //BETTER: unsigned?
Node* p = NULL;
int ccase = 0;
const TBstr* bstr;
if(_index < j) //the right neighbor
{
p = _father->getChild(_index+1);
k = p->getNum();
if((unsigned)k > MIN_KEY_NUM)
ccase = 2;
else //==MIN_KEY_NUM
ccase = 1;
}
if(_index > 0) //the left neighbor
{
Node* tp = _father->getChild(_index-1);
unsigned tk = tp->getNum();
if(ccase < 2)
{
if(ccase == 0)
ccase = 3;
if(tk > MIN_KEY_NUM)
ccase = 4;
}
if(ccase > 2)
{
p = tp;
k = tk;
}
}
switch(ccase)
{
case 1: //union right to this
for(i = 0; i < k; ++i)
{
this->addKey(p->getKey(i), this->getNum());
this->addValue(p->getValue(i), this->getNum());
this->addNum();
}
_father->subKey(_index, true);
_father->subChild(_index+1);
_father->subNum();
this->next = p->getNext();
if(this->next != NULL)
this->next->setPrev(this);
p->setNum(0); //NOTICE: adjust num before delete!
//delete p;
break;
case 2: //move one from right
this->addKey(p->getKey(0), this->getNum());
_father->setKey(p->getKey(1), _index, true);
p->subKey(0);
this->addValue(p->getValue(0), this->getNum());
p->subValue(0);
this->addNum();
p->subNum();
break;
case 3: //union left to this
//BETTER: move all keys/etc one time
for(i = k; i > 0; --i)
{
int t = i - 1;
this->addKey(p->getKey(t), 0);
this->addValue(p->getValue(t), 0);
this->addNum();
}
_father->subKey(_index-1, true);
_father->subChild(_index-1);
_father->subNum();
this->prev = p->getPrev();
if(this->prev != NULL) //else: leaves-list
this->prev->setNext(this);
p->setNum(0);
//delete p;
break;
case 4: //move one from left
bstr = p->getKey(k-1);
p->subKey(k-1);
this->addKey(bstr, 0);
_father->setKey(bstr, _index-1, true);
this->addValue(p->getValue(k-1), 0);
p->subValue(k-1);
this->addNum();
p->subNum();
break;
default:
print("error in coalesce: Invalid case!");
//printf("error in coalesce: Invalid case!");
}
_father->setDirty();
p->setDirty();
this->setDirty();
if(ccase == 1 || ccase == 3)
return p;
else
return NULL;
}
void
LeafNode::release()
{
if(!this->inMem())
return;
unsigned num = this->getNum();
/*
for(int i = 0; i < num; ++i)
{
keys[i].release();
values[i].release();
}
*/
for(unsigned i = num; i < MAX_KEY_NUM; ++i)
{
keys[i].clear();
values[i].clear();
}
delete[] keys;
delete[] values;
}
LeafNode::~LeafNode()
{
release();
}
void
LeafNode::print(string s)
{
#ifdef DEBUG
unsigned num = this->getNum();
Util::showtime();
fputs("Class LeafNode\n", Util::logsfp);
fputs("Message: ", Util::logsfp);
fputs(s.c_str(), Util::logsfp);
fputs("\n", Util::logsfp);
unsigned i;
if(s == "NODE")
{
fprintf(Util::logsfp, "store: %u\tnum: %u\tflag: %u\n", this->store, num, this->flag);
fprintf(Util::logsfp, "prev: %p\tnext: %p\n", this->prev, this->next);
for(i = 0; i < num; ++i)
{
this->keys[i].print("BSTR");
this->values[i].print("BSTR");
}
}
else if(s == "node")
{
fprintf(Util::logsfp, "store: %u\tnum: %u\tflag: %u\n", this->store, num, this->flag);
fprintf(Util::logsfp, "prev: %p\tnext: %p\n", this->prev, this->next);
}
else if(s == "check node")
{
//check the node, if satisfy B+ definition
bool flag = true;
if(num < MIN_KEY_NUM || num > MAX_KEY_NUM)
flag = false;
if(flag)
{
for(i = 1; i < num; ++i)
{
if(keys[i] > keys[i-1])
continue;
else
break;
}
if( i < num)
flag = false;
}
this->print("node");
if(flag)
fprintf(Util::logsfp, "This node is good\n");
else
fprintf(Util::logsfp, "This node is bad\n");
}
else;
#endif
}

53
KVstore/node/LeafNode.h Normal file
View File

@ -0,0 +1,53 @@
/*=============================================================================
# Filename: LeafNode.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:39
# Description: the leaf-node of a B+ tree
=============================================================================*/
#ifndef _LEAF_NODE_H
#define _LEAF_NODE_H
#include "../util/Util.h"
#include "../bstr/TBstr.h"
#include "Node.h"
class LeafNode: public Node
{
protected:
Node* prev; //LeafNode
Node* next;
TBstr* values;
void AllocValues();
//void FreeValues();
public:
LeafNode();
LeafNode(bool isVirtual);
//LeafNode(Storage* TSM);
void Virtual();
void Normal();
Node* getPrev() const;
Node* getNext() const;
const TBstr* getValue(int _index) const;
bool setValue(const TBstr* _value, int _index, bool ifcopy = false);
bool addValue(const TBstr* _value, int _index, bool ifcopy = false);
bool subValue(int _index, bool ifdel = false);
void setPrev(Node* _prev);
void setNext(Node* _next);
unsigned getSize() const;
Node* split(Node* _father, int _index);
Node* coalesce(Node* _father, int _index);
void release();
~LeafNode();
void print(std::string s); //DEBUG
/*non-sense virtual function
Node* getChild(int _index) const;
bool addChild(Node* _child, int _index);
bool subChild(int _index);
*/
};
//BETTER: prev isn't a must, and reverse-range can be achieved using recursive-next
#endif

272
KVstore/node/Node.cpp Normal file
View File

@ -0,0 +1,272 @@
/*=============================================================================
# Filename: Node.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:39
# Description: achieve functions in Node.h
=============================================================================*/
#include "Node.h"
using namespace std;
void
Node::AllocKeys()
{
keys = new TBstr[MAX_KEY_NUM];
}
/*
void
Node::FreeKeys()
{
delete[] keys;
}
*/
Node::Node()
{
store = flag = 0;
flag |= NF_IM;
AllocKeys();
}
Node::Node(bool isVirtual)
{
store = flag = 0;
if(!isVirtual)
{
flag |= NF_IM;
AllocKeys();
}
}
/*
Node::Node(Storage* TSM)
{
AllocKeys();
TSM->readNode(this, Storage::OVER);
}
*/
bool
Node::isLeaf() const
{
return this->flag & NF_IL;
}
bool
Node::isDirty() const
{
return this->flag & NF_ID;
}
void
Node::setDirty()
{
this->flag |= NF_ID;
}
void
Node::delDirty()
{
this->flag &= ~NF_ID;
}
bool
Node::inMem() const
{
return this->flag & NF_IM;
}
void
Node::setMem()
{
this->flag |= NF_IM;
}
void
Node::delMem()
{
this->flag &= ~NF_IM;
}
/*
bool
Node::isVirtual() const
{
return this->flag & NF_IV;
}
void
Node::setVirtual()
{
this->flag |= NF_IV;
}
void
Node::delVirtual()
{
this->flag &= ~NF_IV;
}
*/
unsigned
Node::getRank() const
{
return this->flag & NF_RK;
}
void
Node::setRank(unsigned _rank)
{
this->flag &= ~NF_RK;
this->flag |= _rank;
}
unsigned
Node::getHeight() const
{
return (this->flag & NF_HT)>>20;
}
void
Node::setHeight(unsigned _h)
{
this->flag &= ~NF_HT;
this->flag |= (_h<<20);
}
unsigned
Node::getNum() const
{
return (this->flag & NF_KN)>>12;
}
bool
Node::setNum(int _num)
{
if(_num < 0 || (unsigned)_num > MAX_KEY_NUM)
{
print(string("error in setNum: Invalid num ") + Util::int2string(_num));
return false;
}
this->flag &= ~NF_KN;
this->flag |= (_num<<12);
return true;
}
bool
Node::addNum()
{
if(this->getNum() + 1 > MAX_KEY_NUM)
{
print("error in addNum: Invalid!");
return false;
}
this->flag += (1<<12);
return true;
}
bool
Node::subNum()
{
if(this->getNum() < 1)
{
print("error in subNum: Invalid!");
return false;
}
this->flag -= (1<<12);
return true;
}
unsigned
Node::getStore() const
{
return this->store;
}
void
Node::setStore(unsigned _store)
{
this->store = _store;
}
unsigned
Node::getFlag() const
{
return flag;
}
void
Node::setFlag(unsigned _flag)
{
this->flag = _flag;
}
const TBstr*
Node::getKey(int _index) const
{
int num = this->getNum();
if(_index < 0 || _index >= num)
{
//print(string("error in getKey: Invalid index ") + Util::int2string(_index));
printf("error in getKey: Invalid index\n");
return NULL;
}
else
return this->keys + _index;
}
bool
Node::setKey(const TBstr* _key, int _index, bool ifcopy)
{
int num = this->getNum();
if(_index < 0 || _index >= num)
{
print(string("error in setKey: Invalid index ") + Util::int2string(_index));
return false;
}
if(ifcopy)
keys[_index].copy(_key);
else
keys[_index] = *_key;
return true;
}
bool
Node::addKey(const TBstr* _key, int _index, bool ifcopy)
{
int num = this->getNum();
if(_index < 0 || _index > num)
{
print(string("error in addKey: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
//NOTICE: if num == MAX_KEY_NUM, will visit keys[MAX_KEY_NUM], not legal!!!
//however. tree operations ensure that: when node is full, not add but split first!
for(i = num - 1; i >= _index; --i)
keys[i+1] = keys[i];
if(ifcopy)
keys[_index].copy(_key);
else
keys[_index] = *_key;
return true;
}
bool
Node::subKey(int _index, bool ifdel)
{
int num = this->getNum();
if(_index < 0 || _index >= num)
{
print(string("error in subKey: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
if(ifdel)
keys[_index].release();
for(i = _index; i < num - 1; ++i)
keys[i] = keys[i+1];
return true;
}

108
KVstore/node/Node.h Normal file
View File

@ -0,0 +1,108 @@
/*=============================================================================
# Filename: Node.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:38
# Description: basic Node class, father of IntlNode and LeafNode
=============================================================================*/
#ifndef _NODE_H
#define _NODE_H
#include "../util/Util.h"
#include "../bstr/TBstr.h"
class Node //abstract basic class
{
public:
static const unsigned DEGREE = 2 * 63; //the degree of B+ tree
static const unsigned MAX_CHILD_NUM = DEGREE;
static const unsigned MIN_CHILD_NUM = DEGREE >> 1;
static const unsigned MAX_KEY_NUM = MAX_CHILD_NUM - 1; //max key-num
static const unsigned MIN_KEY_NUM = MIN_CHILD_NUM - 1; //min key-num
/* diffrent flags for tree-nodes, 32-bit put rank in low-bits means no need to move*/
static const unsigned NF_IL = 0x80000000; //is leaf
static const unsigned NF_ID = 0x00080000; //is dirty, in rank-area
static const unsigned NF_IM = 0x20000000; //in memory, not virtual
//static const unsigned NF_IV = 0x10000000; //is virtual
static const unsigned NF_RK = 0x00ffffff; //select-rank, in Storage
static const unsigned NF_HT = 0xf00000; //height area in rank
static const unsigned NF_KN = 0x07f000; //NOTICE: decided by DEGREE
static const unsigned INTL_SIZE = sizeof(TBstr) * MAX_KEY_NUM;
static const unsigned LEAF_SIZE = 2 * INTL_SIZE;
protected:
unsigned store; //store address, the BLock index
unsigned flag; //NF_RK, NF_IL,NF_ID, NF_IV, propety
//int num; //totle keys num
//Node* father; //point to father-node, which must be IntlNode
TBstr* keys;
void AllocKeys();
//void FreeKeys();
public:
Node();
Node(bool isVirtual);
bool isLeaf() const;
bool isDirty() const;
void setDirty();
void delDirty();
bool inMem() const;
void setMem();
void delMem();
//bool isVirtual() const;
//void setVirtual();
//void delVirtual();
unsigned getRank() const;
void setRank(unsigned _rank);
unsigned getHeight() const;
void setHeight(unsigned _h);
unsigned getNum() const;
bool setNum(int _num);
bool addNum();
bool subNum();
unsigned getStore() const;
void setStore(unsigned _store);
unsigned getFlag() const;
void setFlag(unsigned _flag);
const TBstr* getKey(int _index) const; //need to check the index
bool setKey(const TBstr* _key, int _index, bool ifcopy = false);
bool addKey(const TBstr* _key, int _index, bool ifcopy = false);
bool subKey(int _index, bool ifdel = false);
//virtual functions: polymorphic
virtual Node* getChild(int _index) const { return NULL; };
virtual bool setChild(Node* _child, int _index) { return true; };
virtual bool addChild(Node* _child, int _index) { return true; };
virtual bool subChild(int _index) { return true; };
virtual Node* getPrev() const { return NULL; };
virtual Node* getNext() const { return NULL; };
virtual const TBstr* getValue(int _index) const { return NULL; };
virtual bool setValue(const TBstr* _value, int _index, bool ifcopy = false) { return true; };
virtual bool addValue(const TBstr* _value, int _index, bool ifcopy = false) { return true; };
virtual bool subValue(int _index, bool ifdel = false) { return true;};
virtual void setPrev(Node* _prev) {};
virtual void setNext(Node* _next) {};
virtual void Virtual() = 0;
virtual void Normal() = 0;
virtual unsigned getSize() const = 0; //return all memory owned
virtual Node* split(Node* _father, int _index) = 0;
virtual Node* coalesce(Node* _father, int _index) = 0;
virtual void release() = 0; //release the node, only remain necessary information
virtual ~Node() {};
virtual void print(std::string s) = 0; //DEBUG(print the Node)
};
/*NOTICE(operations in release())
*To save memory, we can only remain store and flag(childs added for Leaf).
*However, in this way childs'pointers is ok to change, use Node** or Node*& is also nonsense
*because the pointer variable may die.
*Another way is only to release dynamic memory, and store thw whole, read the TBstr only to
*build. In this way nodes'pointer doesn't change, and operation is simplified, while memory
*is consumed a bit more. Because TBstrs consume the most memory, and memory-disk swapping is
*the most time-consuming thing, it seems to be a better way.
*WARN:when a node is in memory and not deleted, its basic content is always being! If nodes are
*really too many, this will cause disaster because we can't swap them out until tree is closed!
*To solve this problem, there should be two types of release-function: one to release TBstr, one
*to release the whole(pointer is invalid and rebuild problem)
*/
#endif

74
KVstore/rangevalue/RangeValue.cpp Normal file
View File

@ -0,0 +1,74 @@
/*=============================================================================
# Filename: RangeValue.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:42
# Description: achieve functions in RangeValue.h
=============================================================================*/
#include "RangeValue.h"
using namespace std;
RangeValue::RangeValue()
{
this->fp = NULL;
this->transfer.setStr((char*)malloc(1 << 20));
this->transfer_size = 1 << 20;
}
void
RangeValue::reset()
{
fseek(this->fp, 0, SEEK_SET);
}
bool
RangeValue::open()
{
if(this->fp != NULL)
fclose(this->fp);
if((this->fp = fopen("logs/answer.dat", "w+b")) == NULL)
return false;
else
return true;
}
bool
RangeValue::write(const TBstr* _bp)
{
if(this->fp == NULL)
return false;
unsigned len = _bp->getLen();
fwrite(&len, sizeof(unsigned), 1, this->fp);
fwrite(_bp->getStr(), sizeof(char), len, this->fp);
return true;
}
const TBstr*
RangeValue::read()
{
if(this->fp == NULL)
return NULL;
unsigned len = 0;
fread(&len, sizeof(unsigned), 1, this->fp);
if(feof(this->fp))
return NULL; //indicate the end
if(len + 1 > this->transfer_size)
{
transfer.release();
transfer.setStr((char*)malloc(len+1));
this->transfer_size = len + 1;
}
fread(transfer.getStr(), sizeof(char), len, this->fp);
transfer.getStr()[len] = '\0'; //set for string() in KVstore
transfer.setLen(len);
return &transfer;
}
RangeValue::~RangeValue()
{
if(this->fp != NULL)
fclose(this->fp);
//transfer will de deleted as TBstr
}

30
KVstore/rangevalue/RangeValue.h Normal file
View File

@ -0,0 +1,30 @@
/*=============================================================================
# Filename: RangeValue.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:41
# Description: set and deal of ranging values
=============================================================================*/
#ifndef _RANGE_VALUE_H
#define _RANGE_VALUE_H
#endif
#include "../util/Util.h"
#include "../bstr/TBstr.h"
class RangeValue
{
private:
FILE* fp;
TBstr transfer;
unsigned transfer_size;
public:
RangeValue();
void reset();
bool open();
bool write(const TBstr* _bp);
const TBstr* read();
~RangeValue();
};

639
KVstore/storage/Storage.cpp Normal file
View File

@ -0,0 +1,639 @@
/*=============================================================================
# Filename: Storage.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:43
# Description: achieve functions in Storage.h
=============================================================================*/
#include "Storage.h"
using namespace std;
Storage::Storage()
{ //not use ../logs/, notice the location of program
#ifdef DEBUG
if((Util::logsfp = fopen("logs/default.log", "w+")) == NULL)
{
printf("Open error: logs/default.log\n");
Util::logsfp = stdout;
}
#endif
cur_block_num = SET_BLOCK_NUM;
filepath = "";
freelist = NULL;
treefp = NULL;
minheap = NULL;
freemem = MAX_BUFFER_SIZE;
}
Storage::Storage(string& _filepath, string& _mode, unsigned* _height)
{
#ifdef DEBUG
if((Util::logsfp = fopen("logs/default.log", "w+")) == NULL)
{
printf("Open error: logs/default.log\n");
Util::logsfp = stdout;
}
#endif
cur_block_num = SET_BLOCK_NUM; //initialize
this->filepath = _filepath;
if(_mode == string("build"))
treefp = fopen(_filepath.c_str(), "w+b");
else if(_mode == string("open"))
treefp = fopen(_filepath.c_str(), "r+b");
else
{
print(string("error in Storage: Invalid mode ") + _mode);
return;
}
if(treefp == NULL)
{
print(string("error in Storage: Open error ") + _filepath);
return;
}
this->treeheight = _height; //originally set to 0
this->freemem = MAX_BUFFER_SIZE;
this->freelist = new BlockInfo; //null-head
unsigned i, j, k; //j = (SuperNum-1)*BLOCK_SIZE
BlockInfo* bp;
if(_mode == "build")
{ //write basic information
i = 0;
fwrite(&i, sizeof(unsigned), 1, this->treefp); //height
fwrite(&i, sizeof(unsigned), 1, this->treefp); //rootnum
fwrite(&cur_block_num, sizeof(unsigned), 1, this->treefp); //current block num
fseek(this->treefp, BLOCK_SIZE, SEEK_SET);
bp = this->freelist;
j = cur_block_num / 8;
for(i = 0; i < j; ++i)
{
fputc(0, this->treefp);
for(k = 0; k < 8; ++k)
{
bp->next = new BlockInfo(i*8+k+1, NULL);
bp = bp->next;
}
}
}
else //_mode == "open"
{
//read basic information
int rootnum;
char c;
fread(this->treeheight, sizeof(unsigned), 1, this->treefp);
fread(&rootnum, sizeof(unsigned), 1, this->treefp);
fread(&cur_block_num, sizeof(unsigned), 1, this->treefp);
fseek(this->treefp, BLOCK_SIZE, SEEK_SET);
bp = this->freelist;
j = cur_block_num / 8;
for(i = 0; i < j; ++i)
{
c = fgetc(treefp);
for(k = 0; k < 8; ++k)
{
if((c & (1 << k)) == 0)
{
bp->next = new BlockInfo(i*8+7-k+1, NULL);
bp = bp->next;
}
}
}
fseek(treefp, Address(rootnum), SEEK_SET);
//treefp is now ahead of root-block
}
this->minheap = new Heap(HEAP_SIZE);
}
bool
Storage::preRead(Node*& _root, Node*& _leaves_head, Node*& _leaves_tail) //pre-read and build whole tree
{ //set root(in memory) and leaves_head
//TODO: false when exceed memory
_leaves_tail = _leaves_head = _root = NULL;
if(ftell(this->treefp) == 0) //root is null
{
return true;
}
unsigned next, store, j, pos = 0;
unsigned h = *this->treeheight;
Node* p;
//read root node
this->createNode(p);
_root = p;
fread(&next, sizeof(unsigned), 1, treefp);
//use stack to achieve
long address[h]; //current address
unsigned used[h]; //used child num
unsigned total[h]; //total child num
unsigned block[h]; //next block num
Node* nodes[h];
address[pos] = ftell(treefp);
used[pos] = 0;
total[pos]= p->getNum() + 1;
block[pos] = next;
nodes[pos] = p;
pos++;
Node* prev = NULL;
while(pos > 0)
{
j = pos - 1;
if(nodes[j]->isLeaf() || used[j] == total[j]) //LeafNode or ready IntlNode
{
if(nodes[j]->isLeaf())
{
if(prev != NULL)
{
prev->setNext(nodes[j]);
nodes[j]->setPrev(prev);
}
prev = nodes[j];
}
pos--;
continue;
}
fseek(this->treefp, address[j], SEEK_SET);
fread(&store, sizeof(unsigned), 1, treefp);
this->ReadAlign(block + j);
address[j] = ftell(treefp);
fseek(treefp, Address(store), SEEK_SET);
this->createNode(p);
nodes[j]->setChild(p, used[j]);
used[j]++;
fread(&next, sizeof(unsigned), 1, treefp);
address[pos] = ftell(treefp);
used[pos] = 0;
total[pos] = p->getNum() + 1;
block[pos] = next;
nodes[pos] = p;
pos++;
}
//set leaves and read root, which is always keeped in-mem
p = _root;
while(!p->isLeaf())
{
p = p->getChild(0);
}
_leaves_head = p;
p = _root;
while(!p->isLeaf())
{
p = p->getChild(p->getNum());
}
_leaves_tail = p;
int memory = 0;
this->readNode(_root, &memory);
this->request(memory);
return true;
}
long //8-byte in 64-bit machine
Storage::Address(unsigned _blocknum) const //BETTER: inline function
{
if(_blocknum == 0)
return 0;
else if(_blocknum > cur_block_num)
{
//print(string("error in Address: Invalid blocknum ") + Util::int2string(_blocknum));
return -1; //address should be non-negative
}
//NOTICE: here should explictly use long
return (long)(this->SuperNum+_blocknum-1) * (long)BLOCK_SIZE;
}
unsigned
Storage::Blocknum(long address) const
{
return (address/BLOCK_SIZE) + 1 - this->SuperNum;
}
unsigned
Storage::AllocBlock()
{
BlockInfo* p = this->freelist->next;
if(p == NULL)
{
for(unsigned i = 0; i < SET_BLOCK_INC; ++i)
{
cur_block_num++; //BETTER: check if > MAX_BLOCK_NUM
this->FreeBlock(cur_block_num);
}
p = this->freelist->next;
}
unsigned t = p->num;
this->freelist->next = p->next;
delete p;
return t;
}
void
Storage::FreeBlock(unsigned _blocknum)
{ //QUERY: head-sub and tail-add will be better?
BlockInfo* bp = new BlockInfo(_blocknum, this->freelist->next);
this->freelist->next = bp;
}
void
Storage::ReadAlign(unsigned* _next)
{
if(ftell(treefp) % BLOCK_SIZE == 0)
{
fseek(treefp, Address(*_next), SEEK_SET);
fread(_next, sizeof(unsigned), 1, treefp);
}
}
void
Storage::WriteAlign(unsigned* _curnum, bool& _SpecialBlock)
{
if(ftell(treefp) % BLOCK_SIZE == 0)
{
unsigned blocknum = this->AllocBlock();
fseek(treefp, Address(*_curnum), SEEK_SET);
if(_SpecialBlock)
{
fseek(treefp, 4, SEEK_CUR);
_SpecialBlock = false;
}
fwrite(&blocknum, sizeof(unsigned), 1, treefp);
fseek(treefp, Address(blocknum)+4, SEEK_SET);
*_curnum = blocknum;
}
}
bool
Storage::readNode(Node* _np, int* _request)
{
if(_np == NULL || _np->inMem())
return false; //can't read or needn't
fseek(treefp, Address(_np->getStore()), SEEK_SET);
bool flag = _np->isLeaf();
unsigned next;
unsigned i, num = _np->getNum();
TBstr bstr;
fseek(treefp, 4, SEEK_CUR);
fread(&next, sizeof(unsigned), 1, treefp);
//read data, use readTBstr...
//fread(treefp, "%u", &num);
//_np->setNum(num);
if(flag)
*_request += Node::LEAF_SIZE;
else
*_request += Node::INTL_SIZE;
_np->Normal();
if(!flag)
fseek(treefp, 4 * (num + 1), SEEK_CUR);
for(i = 0; i < num; ++i)
{
this->readTBstr(&bstr, &next);
_np->setKey(&bstr, i);
}
if(flag)
{
for(i = 0; i < num; ++i)
{
this->readTBstr(&bstr, &next);
*_request += bstr.getLen();
_np->setValue(&bstr, i);
}
}
//_np->setFlag((_np->getFlag() & ~Node::NF_IV & ~Node::NF_ID) | Node::NF_IM);
//_np->delVirtual();
_np->delDirty();
//_np->setMem();
this->updateHeap(_np, _np->getRank(), false);
bstr.clear();
return true;
}
bool
Storage::createNode(Node*& _np) //cretae virtual nodes, not in-mem
{
/*
if(ftell(this->treefp)== 0) //null root
{
_np = NULL;
return false;
}
*/
unsigned t; //QUERY: maybe next-flag... will be better-storage?
bool flag = false; //IntlNode
fread(&t, sizeof(unsigned), 1, treefp);
if((t & Node::NF_IL) > 0) //WARN: according to setting
flag = true; //LeafNode
if(flag)
{
//this->request(sizeof(LeafNode));
_np = new LeafNode(true);
}
else
{
//this->request(sizeof(IntlNode));
_np = new IntlNode(true);
}
//fseek(treefp, -4, SEEK_CUR);
//_np->setFlag(_np->getFlag() | (t & Node::NF_RK));
//_np->setRank(t);
_np->setFlag(t);
_np->delDirty();
_np->delMem();
_np->setStore(Blocknum(ftell(treefp)-4));
return true;
}
bool
Storage::writeNode(Node* _np)
{
if(_np == NULL || !_np->inMem() || (_np->getRank() > 0 && !_np->isDirty()))
return false; //not need to write back
unsigned num = _np->getNum(), i;
bool flag = _np->isLeaf(), SpecialBlock = true;
/*
if(!flag)
{
for(i = 0; i <= num; ++i)
if(_np->getChild(i)->isDirty())
return false; //NOTICE: all childs must be clean!
}
*/
//to release original blocks
unsigned store = _np->getStore(), next;
//if first store is 0, meaning a new node
fseek(this->treefp, Address(store)+4, SEEK_SET);
fread(&next, sizeof(unsigned), 1, treefp);
while(store != 0)
{
this->FreeBlock(store);
store = next;
fseek(treefp, Address(store), SEEK_SET);
fread(&next, sizeof(unsigned), 1, treefp);
}
if(num == 0)
return true; //node is empty!
unsigned t;
//write Node information
unsigned blocknum = this->AllocBlock();
_np->setStore(blocknum);
long address = this->Address(blocknum);
fseek(this->treefp, address, SEEK_SET);
t = _np->getFlag();
fwrite(&t, sizeof(unsigned), 1, treefp); //DEBUG
fseek(treefp, 4, SEEK_CUR);
if(!flag)
{
for(i = 0; i <= num; ++i)
{
t = _np->getChild(i)->getStore();
fwrite(&t, sizeof(unsigned), 1, treefp); //DEBUG
this->WriteAlign(&blocknum, SpecialBlock);
}
}
for(i = 0; i < num; ++i)
this->writeTBstr(_np->getKey(i), &blocknum, SpecialBlock);
if(flag)
{
for(i = 0; i < num; ++i)
this->writeTBstr(_np->getValue(i), &blocknum, SpecialBlock);
}
fseek(treefp, Address(blocknum), SEEK_SET);
if(SpecialBlock)
fseek(treefp, 4, SEEK_CUR);
t = 0;
fwrite(&t, sizeof(unsigned), 1, treefp); //the end-block
//_np->setFlag(_np->getFlag() & ~Node::NF_ID);
_np->delDirty();
return true;
}
bool
Storage::readTBstr(TBstr* _bp, unsigned* _next)
{
//long address;
unsigned len, i, j;
fread(&len, sizeof(unsigned), 1, this->treefp);
this->ReadAlign(_next);
//this->request(len);
char* s = (char*)malloc(len);
_bp->setLen(len);
for(i = 0; i + 4 < len; i += 4)
{
fread(s+i, sizeof(char), 4, treefp);
this->ReadAlign(_next);
}
while(i < len)
{
fread(s+i, sizeof(char), 1, treefp); //BETTER
i++;
}
j = len % 4;
if(j > 0)
j = 4 - j;
fseek(treefp, j, SEEK_CUR);
this->ReadAlign(_next);
_bp->setStr(s);
return true;
}
bool
Storage::writeTBstr(const TBstr* _bp, unsigned* _curnum, bool& _SpecialBlock)
{
unsigned i, j, len = _bp->getLen();
fwrite(&len, sizeof(unsigned), 1, treefp);
this->WriteAlign(_curnum, _SpecialBlock);
char* s = _bp->getStr();
for(i = 0; i + 4 < len; i += 4)
{
fwrite(s+i, sizeof(char), 4, treefp);
this->WriteAlign(_curnum, _SpecialBlock);
}
while(i < len)
{
fwrite(s+i, sizeof(char), 1, treefp);
i++;
}
j = len % 4;
if(j > 0)
j = 4 - j;
fseek(treefp, j, SEEK_CUR);
this->WriteAlign(_curnum, _SpecialBlock);
return true;
}
bool
Storage::writeTree(Node* _root) //write the whole tree back and close treefp
{
fseek(this->treefp, 0, SEEK_SET);
fwrite(this->treeheight, sizeof(unsigned), 1, treefp);
//delete all nonsense-node in heap, otherwise will waste storage permanently
Node* p;
while(1)
{ //all non-sense nodes will be in-head-area, due to minimal rank
p = minheap->getTop();
if(p == NULL) //heap is empty, only when root==NULL
break;
if(p->getRank() == 0) //indicate non-sense node
{
this->minheap->remove();
this->writeNode(p);
delete p;
}
else
break;
}
unsigned i, j, t;
//QUERY: another way to write all nodes back is to print out all nodes in heap
//but this method will cause no node in heap any more, while operations may be
//afetr tree-saving. Which method is better?
//write nodes recursively using stack, including root-num
if(_root != NULL)
{
Node* p = _root;
unsigned h = *this->treeheight, pos = 0;
Node* ns[h];
int ni[h];
ns[pos] = p;
ni[pos] = p->getNum();
pos++;
while(pos > 0)
{
j = pos - 1;
p = ns[j];
if(p->isLeaf() || ni[j] < 0) //leaf or all childs are ready
{
this->writeNode(p);
pos--;
continue;
}
ns[pos] = p->getChild(ni[j]);
ni[pos] = ns[pos]->getNum();
pos++;
ni[j]--;
}
t = _root->getStore();
}
else
t = 0;
fseek(this->treefp, 4, SEEK_SET);
fwrite(&t, sizeof(unsigned), 1, treefp); //write the root num
fwrite(&cur_block_num, sizeof(unsigned), 1, treefp);//write current blocks num
fseek(treefp, BLOCK_SIZE, SEEK_SET);
j = cur_block_num / 8; //(SuperNum-1)*BLOCK_SIZE;
//reset to 1 first
for(i = 0; i < j; ++i)
{
fputc(0xff, treefp);
}
char c;
BlockInfo* bp = this->freelist->next;
while(bp != NULL)
{
//if not-use then set 0, aligned to byte!
#ifdef DEBUG
if(bp->num > cur_block_num)
{
printf("blocks num exceed, cur_block_num: %u\n", cur_block_num);
exit(1);
}
#endif
j = bp->num - 1;
i = j / 8;
j = 7 - j % 8;
fseek(treefp, BLOCK_SIZE+i, SEEK_SET);
c = fgetc(treefp);
fseek(treefp, -1, SEEK_CUR);
fputc(c & ~(1 << j), treefp);
bp = bp->next;
}
//fclose(this->treefp);
return true;
}
void
Storage::updateHeap(Node* _np, unsigned _rank, bool _inheap) const
{
if(_inheap) //already in heap, to modify
{
unsigned t = _np->getRank();
_np->setRank(_rank);
if(t < _rank)
this->minheap->modify(_np, false);
else if(t > _rank)
this->minheap->modify(_np, true);
else;
}
else //not in heap, to add
{
_np->setRank(_rank);
this->minheap->insert(_np);
}
}
void
Storage::request(int _needmem) //aligned to byte
{ //NOTICE: <0 means release
if(_needmem > 0 && this->freemem < (unsigned)_needmem)
if(!this->handler(_needmem - freemem)) //disaster in buffer memory
{
print(string("error in request: out of buffer-mem, now to exit"));
exit(1);
}
this->freemem -= _needmem;
}
bool
Storage::handler(unsigned _needmem) //>0
{
Node* p;
unsigned size;
//if(_needmem < SET_BUFFER_SIZE) //to recover to SET_BUFFER_SIZE buffer
// _needmem = SET_BUFFER_SIZE;
while(1)
{
p = this->minheap->getTop();
if(p == NULL)
return false; //can't satisfy or can't recover to SET_BUFFER_SIZE
this->minheap->remove();
size = p->getSize();
this->freemem += size;
this->writeNode(p);
if(p->getNum() > 0)
p->Virtual();
else
delete p; //non-sense node
if(_needmem > size)
_needmem -= size;
else
break;
}
return true;
}
Storage::~Storage()
{
//release heap and freelist...
BlockInfo* bp = this->freelist;
BlockInfo* next;
while(bp != NULL)
{
next = bp->next;
delete bp;
bp = next;
}
delete this->minheap;
fclose(this->treefp);
#ifdef DEBUG
fclose(Util::logsfp);
#endif
}
void
Storage::print(string s)
{
#ifdef DEBUG
Util::showtime();
fputs("Class Storage\n", Util::logsfp);
fputs("Message: ", Util::logsfp);
fputs(s.c_str(), Util::logsfp);
fputs("\n", Util::logsfp);
#endif
}

73
KVstore/storage/Storage.h Normal file
View File

@ -0,0 +1,73 @@
/*=============================================================================
# Filename: Storage.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:43
# Description: swap between memory and disk, achieving system-like method
=============================================================================*/
#ifndef _STORAGE_H
#define _STORAGE_H
#include "../util/Util.h"
#include "../node/Node.h"
#include "../node/IntlNode.h"
#include "../node/LeafNode.h"
#include "../heap/Heap.h"
#include "../bstr/TBstr.h"
#include "file.h"
//It controls read, write, swap
class Storage
{
public:
static const unsigned BLOCK_SIZE = 1 << 16; //fixed size of disk-block
static const unsigned long long MAX_BUFFER_SIZE = 0x1ffffffff; //max buffer size
//static const unsigned SET_BUFFER_SIZE = 1 << 30; //set buffer size
static const unsigned HEAP_SIZE = MAX_BUFFER_SIZE/Node::INTL_SIZE;
static const unsigned MAX_BLOCK_NUM = 1 << 24; //max block-num
//below two constants: must can be exactly divided by 8
static const unsigned SET_BLOCK_NUM = 1 << 8; //initial blocks num
static const unsigned SET_BLOCK_INC = SET_BLOCK_NUM; //base of blocks-num inc
static const unsigned SuperNum = MAX_BLOCK_NUM/(8*BLOCK_SIZE)+1;
//static const unsigned TRANSFER_CAPACITY = BLOCK_SIZE;
//enum ReadType { OVER = 0, EXPAND, NORMAL };
private:
unsigned cur_block_num;
std::string filepath;
unsigned* treeheight;
BlockInfo* freelist;
FILE* treefp; //file: tree nodes
Heap* minheap; //heap of Nodes's pointer, sorted in NF_RK
//NOTICE: freemem's type is long long here, due to large memory in server.
//However, needmem in handler() and request() is ok to be int/unsigned.
//Because the bstr' size is controlled, so is the node.
unsigned long long freemem; //free memory to use, non-negative
//unsigned long long time; //QUERY(achieving an old-swap startegy?)
long Address(unsigned _blocknum) const;
unsigned Blocknum(long address) const;
unsigned AllocBlock();
void FreeBlock(unsigned _blocknum);
void ReadAlign(unsigned* _next);
void WriteAlign(unsigned* _next, bool& _SpecialBlock);
public:
Storage();
Storage(std::string& _filepath, std::string& _mode, unsigned* _height);//create a fixed-size file or open an existence
bool preRead(Node*& _root, Node*& _leaves_head, Node*& _leaves_tail); //read and build all nodes, only root in memory
bool readNode(Node* _np, int* _request); //read, if virtual
bool createNode(Node*& _np); //use fp to create a new node
//NOTICE(if children and child not exist, build children's Nodes)
bool writeNode(Node* _np);
bool readTBstr(TBstr* _bp, unsigned* _next);
bool writeTBstr(const TBstr* _bp, unsigned* _curnum, bool& _SpecialBlock);
bool writeTree(Node* _np);
void updateHeap(Node* _np, unsigned _rank, bool _inheap) const;
void request(int _needmem); //deal with memory request
bool handler(unsigned _needmem); //swap some nodes out
//bool update(); //update InMem Node's rank, with clock
~Storage();
void print(std::string s); //DEBUG
};
#endif

89
KVstore/storage/file.h Normal file
View File

@ -0,0 +1,89 @@
/*=============================================================================
# Filename: file.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:42
# Description: disk file memlayout
=============================================================================*/
#ifndef _FILE_H
#define _FILE_H
#include "../util/Util.h"
/******** manage the disk-file as Blocks-list ********/
/* All blocks of given file are viewed as an array, extended dynamicly
* The first block(0, super-block) includes the information
* about the whole file(for example, the root Node's Block),
* especially a bitset use[MAXBN] which is used to recognize which
* block is in use
* In practical, the normal block numbers from 1 to MAXBN-1,
* so 0 can be used as division.
* (normal block is made of header and data)
* When file opened, program must read this bitset and create an
* freelist(several 10Ms memory),
* and remember to write back the bitset when closing.
* We store each Tree-Node as a unit, which may contain several
* blocks, not requiring continuous.
* While the tree is not closed, better to keep root Node in
* memory all the time.
*/
/*
struct Header
{ //this is the header information at the
//beginning of each block, then the data
//
//If this the first block of a node, we must also store
* the necessary information about the node. For example,
* a bit indicates whether a leaf-node, deciding how it
* should be read. The first block's prev and the
//final block's next should be 0
//blockaddr_t prev;
blockaddr_t next; //WARN(maybe larger type!)
//unsigned short end; //valid data:0~end
};
struct SuperBlock //SuperNum blocks, numbered 0
{
unsigned height;
unsigned rootnum; //use a whole block, may store other information
//for example, nodes's num
char use[BNWD]; //exactly SuperNum-1 blocks
};
//numbered from 1 to MAX_BLOCK_NUM
struct Node //may use several blocks, not must continuously
{
unsigned flag; //only in first block, special-block
unsigned next; //each real data-block, 0 means the end
information:
unsigned num;
unsigned childs[]; //only in IntlNodes
Bstr keys[];
Bstr values[]; //only in LeafNodes
};
*/
//When stored in disk, every Node* pointer should be changed to block-address
//(a bit indicates whether a leaf!),
//and char* should be changed to the real string.
class BlockInfo
{
public:
unsigned num;
BlockInfo* next;
BlockInfo()
{
num = 0;
next = NULL;
}
BlockInfo(unsigned _num, BlockInfo* _bp)
{
num = _num;
next = _bp;
}
};
#endif

596
KVstore/tree/Tree.cpp Normal file
View File

@ -0,0 +1,596 @@
/*=============================================================================
# Filename: Tree.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:45
# Description: achieve functions in Tree.h
=============================================================================*/
#include "Tree.h"
using namespace std;
//tree's operations should be atom(if read nodes)
//sum the request and send to Storage at last
//ensure that all nodes operated are in memory
int request = 0;
Tree::Tree()
{
height = 0;
mode = "";
root = NULL;
leaves_head = NULL;
leaves_tail = NULL;
TSM = NULL;
storepath = "";
filename = "";
transfer_size[0] = transfer_size[1] = transfer_size[2] = 0;
}
Tree::Tree(const string& _storepath, const string& _filename, const char* _mode)
{
storepath = _storepath;
filename = _filename;
this->height = 0;
this->mode = string(_mode);
string filepath = this->getFilePath();
TSM = new Storage(filepath, this->mode, &this->height);
if(this->mode == "open")
this->TSM->preRead(this->root, this->leaves_head, this->leaves_tail);
else
this->root = NULL;
this->transfer[0].setStr((char*)malloc(1 << 20));
this->transfer[1].setStr((char*)malloc(1 << 20));
this->transfer[2].setStr((char*)malloc(1 << 20));
this->transfer_size[0] = this->transfer_size[1] = this->transfer_size[2] = 1 << 20; //initialied to 1M
}
string
Tree::getFilePath()
{
return storepath+"/"+filename;
}
void //WARN: not check _str and _len
Tree::CopyToTransfer(const char* _str, unsigned _len, unsigned _index)
{
if(_index > 2)
return;
/*
if(_str == NULL || _len == 0)
{
printf("error in CopyToTransfer: empty string\n");
return;
}
*/
//unsigned length = _bstr->getLen();
unsigned length = _len;
if(length + 1 > this->transfer_size[_index])
{
transfer[_index].release();
transfer[_index].setStr((char*)malloc(length+1));
this->transfer_size[_index] = length + 1; //one more byte: convenient to add \0
}
memcpy(this->transfer[_index].getStr(), _str, length);
this->transfer[_index].getStr()[length] = '\0'; //set for string() in KVstore
this->transfer[_index].setLen(length);
}
unsigned
Tree::getHeight() const
{
return this->height;
}
void
Tree::setHeight(unsigned _h)
{
this->height = _h;
}
Node*
Tree::getRoot() const
{
return this->root;
}
void
Tree::prepare(Node* _np) const
{
bool flag = _np->inMem();
if(!flag)
this->TSM->readNode(_np, &request); //readNode deal with request
}
bool
Tree::search(const char* _str1, unsigned _len1, char*& _str2, int& _len2)
{
const TBstr* value = NULL;
if(_str1 == NULL || _len1 == 0)
{
printf("error in Tree-search: empty string\n");
return false;
}
this->CopyToTransfer(_str1, _len1, 1);
bool ret = this->search(&transfer[1], value);
if(ret)
{
_str2 = value->getStr();
_len2 = value->getLen();
}
return ret;
}
bool
Tree::search(const TBstr* _key, const TBstr*& _value)
{
request = 0;
TBstr bstr = *_key; //not to modify its memory
int store;
Node* ret = this->find(_key, &store, false);
if(ret == NULL || store == -1 || bstr != *(ret->getKey(store))) //tree is empty or not found
{
bstr.clear();
return false;
}
const TBstr* val = ret->getValue(store);
this->CopyToTransfer(val->getStr(), val->getLen(), 0); //not sum to request
_value = &transfer[0];
this->TSM->request(request);
bstr.clear();
return true;
}
bool
Tree::insert(const char* _str1, unsigned _len1, const char* _str2, unsigned _len2)
{
if(_str1 == NULL || _len1 == 0)
{
printf("error in Tree-insert: empty string\n");
return false;
}
this->CopyToTransfer(_str1, _len1, 1);
this->CopyToTransfer(_str2, _len2, 2); //not check value
bool ret = this->insert(&transfer[1], &transfer[2]);
return ret;
}
bool
Tree::insert(const TBstr* _key, const TBstr* _value)
{
request = 0;
Node* ret;
if(this->root == NULL) //tree is empty
{
leaves_tail = leaves_head = root = new LeafNode;
request += Node::LEAF_SIZE;
this->height = 1;
root->setHeight(1); //add to heap later
}
//this->prepare(this->root); //root must be in-mem
if(root->getNum() == Node::MAX_KEY_NUM)
{
Node* father = new IntlNode;
request += Node::INTL_SIZE;
father->addChild(root, 0);
ret = root->split(father, 0);
if(ret->isLeaf() && ret->getNext() == NULL)
this->leaves_tail = ret;
if(ret->isLeaf())
request += Node::LEAF_SIZE;
else
request += Node::INTL_SIZE;
this->height++; //height rises only when root splits
//WARN: height area in Node: 4 bit!
father->setHeight(this->height); //add to heap later
this->TSM->updateHeap(ret, ret->getRank(), false);
this->root = father;
}
Node* p = this->root;
Node* q;
int i, j;
TBstr bstr = *_key;
while(!p->isLeaf())
{
j = p->getNum();
for(i = 0; i < j; ++i)
if(bstr < *(p->getKey(i)))
break;
q = p->getChild(i);
this->prepare(q);
if(q->getNum() == Node::MAX_KEY_NUM)
{
ret = q->split(p, i);
if(ret->isLeaf() && ret->getNext() == NULL)
this->leaves_tail = ret;
if(ret->isLeaf())
request += Node::LEAF_SIZE;
else
request += Node::INTL_SIZE;
//BETTER: in loop may update multiple times
this->TSM->updateHeap(ret, ret->getRank(), false);
this->TSM->updateHeap(q, q->getRank(), true);
this->TSM->updateHeap(p, p->getRank(), true);
if(bstr < *(p->getKey(i)))
p = q;
else
p = ret;
}
else
{
p->setDirty();
this->TSM->updateHeap(p, p->getRank(), true);
p = q;
}
}
j = p->getNum();
for(i = 0; i < j; ++i)
if(bstr < *(p->getKey(i)))
break;
//insert existing key is ok, but not inserted in
//however, the tree-shape may change due to possible split in former code
bool ifexist = false;
if(i > 0 && bstr == *(p->getKey(i-1)))
ifexist = true;
else
{
p->addKey(_key, i, true);
p->addValue(_value, i, true);
p->addNum();
request += (_key->getLen() + _value->getLen());
p->setDirty();
this->TSM->updateHeap(p, p->getRank(), true);
//_key->clear();
//_value->clear();
}
this->TSM->request(request);
bstr.clear(); //NOTICE: must be cleared!
return !ifexist; //QUERY(which case:return false)
}
bool
Tree::modify(const char* _str1, unsigned _len1, const char* _str2, unsigned _len2)
{
if(_str1 == NULL || _len1 == 0)
{
printf("error in Tree-modify: empty string\n");
return false;
}
this->CopyToTransfer(_str1, _len1, 1);
this->CopyToTransfer(_str2, _len2, 2); //not check value
bool ret = this->modify(&transfer[1], &transfer[2]);
return ret;
}
bool
Tree::modify(const TBstr* _key, const TBstr* _value)
{
request = 0;
TBstr bstr = *_key;
int store;
Node* ret = this->find(_key, &store, true);
if(ret == NULL || store == -1 || bstr != *(ret->getKey(store))) //tree is empty or not found
{
bstr.clear();
return false;
}
unsigned len = ret->getValue(store)->getLen();
ret->setValue(_value, store, true);
request += (_value->getLen()-len);
//_value->clear();
ret->setDirty();
this->TSM->request(request);
bstr.clear();
return true;
}
/* this function is useful for search and modify, and range-query */
Node* //return the first key's position that >= *_key
Tree::find(const TBstr* _key, int* _store, bool ifmodify) const
{ //to assign value for this->bstr, function shouldn't be const!
if(this->root == NULL)
return NULL; //Tree Is Empty
Node* p = root;
int i, j;
TBstr bstr = *_key; //local Bstr: multiple delete
while(!p->isLeaf())
{
if(ifmodify)
p->setDirty();
j = p->getNum();
for(i = 0; i < j; ++i) //BETTER(Binary-Search)
if(bstr < *(p->getKey(i)))
break;
p = p->getChild(i);
this->prepare(p);
}
j = p->getNum();
for(i = 0; i < j; ++i)
if(bstr <= *(p->getKey(i)))
break;
if(i == j)
*_store = -1; //Not Found
else
*_store = i;
bstr.clear();
return p;
}
/*
Node*
Tree::find(unsigned _len, const char* _str, int* store) const
{
}
*/
bool
Tree::remove(const char* _str, unsigned _len)
{
if(_str == NULL || _len == 0)
{
printf("error in Tree-remove: empty string\n");
return false;
}
this->CopyToTransfer(_str, _len, 1);
bool ret = this->remove(&transfer[1]);
return ret;
}
bool //BETTER: if not found, the road are also dirty! find first?
Tree::remove(const TBstr* _key)
{
request = 0;
Node* ret;
if(this->root == NULL) //tree is empty
return false;
Node* p = this->root;
Node* q;
int i, j;
TBstr bstr = *_key;
while(!p->isLeaf())
{
j = p->getNum();
for(i = 0; i < j; ++i)
if(bstr < *(p->getKey(i)))
break;
q = p->getChild(i);
this->prepare(q);
if(q->getNum() < Node::MIN_CHILD_NUM) //==MIN_KEY_NUM
{
if(i > 0)
this->prepare(p->getChild(i-1));
if(i < j)
this->prepare(p->getChild(i+1));
ret = q->coalesce(p, i);
if(ret != NULL)
this->TSM->updateHeap(ret, 0, true);//non-sense node
this->TSM->updateHeap(q, q->getRank(), true);
if(q->isLeaf())
{
if(q->getPrev() == NULL)
this->leaves_head = q;
if(q->getNext() == NULL)
this->leaves_tail = q;
}
if(p->getNum() == 0) //root shrinks
{
//this->leaves_head = q;
this->root = q;
this->TSM->updateHeap(p, 0, true); //instead of delete p
this->height--;
}
}
else
p->setDirty();
this->TSM->updateHeap(p, p->getRank(), true);
p = q;
}
bool flag = false;
j = p->getNum(); //LeafNode(maybe root)
for(i = 0; i < j; ++i)
if(bstr == *(p->getKey(i)))
{
request -= p->getKey(i)->getLen();
request -= p->getValue(i)->getLen();
p->subKey(i, true); //to release
p->subValue(i, true); //to release
p->subNum();
if(p->getNum() == 0) //root leaf 0 key
{
this->root = NULL;
this->leaves_head = NULL;
this->leaves_tail = NULL;
this->height = 0;
this->TSM->updateHeap(p, 0, true); //instead of delete p
}
p->setDirty();
flag = true;
break;
}
this->TSM->request(request);
bstr.clear();
return flag; //i == j, not found
}
const TBstr*
Tree::getRangeValue()
{
return this->VALUES.read();
}
bool //special case: not exist, one-edge-case
Tree::range_query(const TBstr* _key1, const TBstr* _key2)
{ //the range is: *_key1 <= x < *_key2
/*
if(_key1 == NULL && _key2 == NULL)
return false;
*/
//ok to search one-edge, requiring only one be NULL
this->VALUES.open();
/* find and write value */
int store1, store2;
Node *p1, *p2;
if(_key1 != NULL)
{
request = 0;
p1 = this->find(_key1, &store1, false);
if(p1 == NULL || store1 == -1)
return false; //no element
this->TSM->request(request);
}
else
{
p1 = this->leaves_head;
store1 = 0;
}
if(_key2 != NULL)
{ //QUERY: another strategy is to getnext and compare every time to tell end
request = 0;
p2 = this->find(_key2, &store2, false);
if(p2 == NULL)
return false;
else if(store2 == -1)
store2 = p2->getNum();
else if(store2 == 0)
{
p2 = p2->getPrev();
if(p2 == NULL)
return false; //no element
store2 = p2->getNum();
}
this->TSM->request(request);
}
else
{
p2 = this->leaves_tail;
store2 = p2->getNum();
}
Node* p = p1;
unsigned i, l, r;
while(1)
{
request = 0;
this->prepare(p);
if(p == p1)
l = store1;
else
l = 0;
if(p == p2)
r = store2;
else
r = p->getNum();
for(i = l; i < r; ++i)
this->VALUES.write(p->getValue(i));
this->TSM->request(request);
if(p != p2)
p = p->getNext();
else
break;
}
this->VALUES.reset();
return true;
}
bool
Tree::save() //save the whole tree to disk
{
if(TSM->writeTree(this->root))
return true;
else
return false;
}
void
Tree::release(Node* _np) const
{
if(_np == NULL) return;
if(_np->isLeaf())
{
delete _np;
return;
}
int cnt = _np->getNum();
for(; cnt >= 0; --cnt)
release(_np->getChild(cnt));
delete _np;
}
Tree::~Tree()
{
//delete VALUES;
delete TSM;
//recursively delete each Node
release(root);
}
void
Tree::print(string s)
{
#ifdef DEBUG
Util::showtime();
fputs("Class Tree\n", Util::logsfp);
fputs("Message: ", Util::logsfp);
fputs(s.c_str(), Util::logsfp);
fputs("\n", Util::logsfp);
fprintf(Util::logsfp, "Height: %d\n", this->height);
if(s == "tree" || s == "TREE")
{
if(this->root == NULL)
{
fputs("Null Tree\n", Util::logsfp);
return;
}
Node** ns = new Node*[this->height];
int* ni = new int[this->height];
Node* np;
int i, pos = 0;
ns[pos] = this->root;
ni[pos] = this->root->getNum();
pos++;
while(pos > 0)
{
np = ns[pos-1];
i = ni[pos-1];
this->prepare(np);
if(np->isLeaf() || i < 0) //LeafNode or ready IntlNode
{ //child-num ranges: 0~num
if(s == "tree")
np->print("node");
else
np->print("NODE"); //print full node-information
pos--;
continue;
}
else
{
ns[pos] = np->getChild(i);
ni[pos-1]--;
ni[pos] = ns[pos]->getNum();
pos++;
}
}
delete[] ns;
delete[] ni;
}
else if(s == "LEAVES" || s == "leaves")
{
Node* np;
for(np = this->leaves_head; np != NULL; np = np->getNext())
{
this->prepare(np);
if(s == "leaves")
np->print("node");
else
np->print("NODE");
}
}
else if(s == "check tree")
{
//check the tree, if satisfy B+ definition
//TODO
}
else;
#endif
}

75
KVstore/tree/Tree.h Normal file
View File

@ -0,0 +1,75 @@
/*=============================================================================
# Filename: Tree.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:44
# Description: struct and interface of the B+ tree
=============================================================================*/
#ifndef _TREE_H
#define _TREE_H
#include "../util/Util.h"
#include "../node/Node.h"
#include "../node/IntlNode.h"
#include "../node/LeafNode.h"
#include "../storage/Storage.h"
#include "../rangevalue/RangeValue.h"
class Tree
{
private:
unsigned int height; //0 indicates an empty tree
Node* root;
Node* leaves_head; //the head of LeafNode-list
Node* leaves_tail; //the tail of LeafNode-list
std::string mode; //BETTER(to use enum)
Storage* TSM; //Tree-Storage-Manage
RangeValue VALUES; //just for range query
//always alloc one more byte than length, then user can add a '\0'
//to get a real string, instead of new and copy
//other operations will be harmful to search, so store value in
//transfer temporally, while length adjusted.
//TODO: in multi-user case, multiple-search will cause problem,
//so lock is a must. Add lock to transfer is better than to add
//lock to every key/value. However, modify requires a lock for a
//key/value, and multiple search for different keys are ok!!!
TBstr transfer[3]; //0:transfer value searched; 1:copy key-data from const char*; 2:copy val-data from const char*
unsigned transfer_size[3];
std::string storepath;
std::string filename; //ok for user to change
/* some private functions */
std::string getFilePath(); //in UNIX system
void CopyToTransfer(const char* _str, unsigned _len, unsigned _index);
void release(Node* _np) const;
void prepare(Node* _np) const;
public:
Tree(); //always need to initial transfer
Tree(const std::string& _storepath, const std::string& _filename, const char* _mode);
unsigned int getHeight() const;
void setHeight(unsigned _h);
Node* getRoot() const;
//void setRoot(Node* _root);
//insert, search, remove, set
bool search(const char* _str1, unsigned _len1, char*& _str2, int& _len2);
bool search(const TBstr* _key1, const TBstr*& _value);
bool insert(const TBstr* _key, const TBstr* _value);
bool insert(const char* _str1, unsigned _len1, const char* _str2, unsigned _len2);
bool modify(const TBstr* _key, const TBstr* _value);
bool modify(const char* _str1, unsigned _len1, const char* _str2, unsigned _len2);
Node* find(const TBstr* _key, int* store, bool ifmodify) const;
//Node* find(unsigned _len, const char* _str, int* store) const;
bool remove(const TBstr* _key);
bool remove(const char* _str, unsigned _len);
const TBstr* getRangeValue();
bool range_query(const TBstr* _key1, const TBstr* _key2);
bool save();
~Tree();
void print(std::string s); //DEBUG(print the tree)
};
//NOTICE: need to save tree manually before delete, otherwise will cause problem.
//(problem range between two extremes: not-modified, totally-modified)
//After saved, it's ok to continue operations on tree!
#endif

92
KVstore/util/Util.cpp Normal file
View File

@ -0,0 +1,92 @@
/*=============================================================================
# Filename: Util.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:46
# Description: achieve functions in Util.h
=============================================================================*/
#include "Util.h"
using namespace std;
FILE* Util::logsfp; //assigned by Storage
int
Util::compare(const char* _str1, unsigned _len1, const char* _str2, unsigned _len2)
{
int ifswap = 1; //1 indicate: not swapped
if(_len1 > _len2)
{
const char* str = _str1;
_str1 = _str2;
_str2 = str;
unsigned len = _len1;
_len1 = _len2;
_len2 = len;
ifswap = -1;
}
unsigned i;
//DEBUG: if char can be negative, which cause problem when comparing(128+)
//
//NOTICE:little-endian-storage, when string buffer poniter is changed to
//unsigned long long*, the first char is the lowest byte!
/*
unsigned long long *p1 = (unsigned long long*)_str1, *p2 = (unsigned long long*)_str2;
unsigned limit = _len1/8;
for(i = 0; i < limit; ++i, ++p1, ++p2)
{
if((*p1 ^ *p2) == 0) continue;
else
{
if(*p1 < *p2) return -1 * ifswap;
else return 1 * ifswap;
}
}
for(i = 8 * limit; i < _len1; ++i)
{
if(_str1[i] < _str2[i]) return -1 * ifswap;
else if(_str1[i] > _str2[i]) return 1 * ifswap;
else continue;
}
if(i == _len2) return 0;
else return -1 * ifswap;
*/
for(i = 0; i < _len1; ++i)
{ //ASCII: 0~127 but c: 0~255(-1) all transfered to unsigned char when comparing
if((unsigned char)_str1[i] < (unsigned char)_str2[i])
return -1 * ifswap;
else if((unsigned char)_str1[i] > (unsigned char)_str2[i])
return 1 * ifswap;
else;
}
if(i == _len2)
return 0;
else
return -1 * ifswap;
}
int
Util::string2int(string s)
{
return atoi(s.c_str());
}
string
Util::int2string(int n)
{
string s;
stringstream ss;
ss<<n;
ss>>s;
return s;
}
void
Util::showtime()
{
fputs("\n\n", logsfp);
time_t now;
time(&now);
fputs(ctime(&now), logsfp);
}

45
KVstore/util/Util.h Normal file
View File

@ -0,0 +1,45 @@
/*=============================================================================
# Filename: Util.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:45
# Description: common macros, functions, classes, etc.
=============================================================================*/
#ifndef _UTIL_H
#define _UTIL_H
/* basic macros and types are defined here, including common headers */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <time.h>
#include <string>
#include <sstream>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
//#define DEBUG 1 //indicate that in debug mode
/******** all static&universal constants and fucntions ********/
class Util
{
public:
/******** this are for debugging ********/
//to build logs-system, each class: print() in time
static FILE* logsfp; //file: executing logs
public:
static int compare(const char* _str1, unsigned _len1, const char* _str2, unsigned _len2); //QUERY(how to use default args)
static int string2int(std::string s);
static std::string int2string(int n);
//string2str: s.c_str()
//str2string: string(str)
static void showtime();
};
#endif

96
Makefile
View File

@ -1,103 +1,104 @@
CC = g++
CFLAGS = -c -g -Wall
objdir=objs/ objdir=objs/
objfile= $(objdir)Bstr.o $(objdir)Database.o $(objdir)KVstore.o $(objdir)Btree.o \
$(objdir)CBtreeFunc.o $(objdir)SPARQLquery.o $(objdir)BasicQuery.o $(objdir)ResultSet.o \ btreeobj = $(objdir)Tree.o $(objdir)Storage.o $(objdir)Node.o $(objdir)IntlNode.o $(objdir)LeafNode.o $(objdir)TBstr.o $(objdir)Util.o $(objdir)Heap.o $(objdir)Hash.o $(objdir)RangeValue.o
$(objdir)SigEntry.o $(objdir)Signature.o $(objdir)Triple.o $(objdir)util.o $(objdir)VSTree.o \
$(objdir)IDList.o $(objdir)EntryBuffer.o $(objdir)LRUCache.o $(objdir)VNode.o $(objdir)DBparser.o \ objfile=$(objdir)Bstr.o $(objdir)Database.o $(objdir)KVstore.o $(objdir)SPARQLquery.o $(objdir)BasicQuery.o \
$(objdir)SparqlParser.o $(objdir)SparqlLexer.o $(objdir)Operation.o $(objdir)Socket.o \ $(objdir)ResultSet.o $(objdir)SigEntry.o $(objdir)Signature.o $(objdir)Triple.o $(objdir)util.o \
$(objdir)Server.o $(objdir)Client.o \ $(objdir)VSTree.o $(objdir)IDList.o $(objdir)EntryBuffer.o $(objdir)LRUCache.o $(objdir)VNode.o \
$(objdir)TurtleParser.o $(objdir)RDFParser.o $(objdir)DBparser.o $(objdir)SparqlParser.o $(objdir)SparqlLexer.o $(objdir)Operation.o \
$(objdir)Socket.o $(objdir)Server.o $(objdir)Client.o $(objdir)TurtleParser.o $(objdir)RDFParser.o \
$(btreeobj)
inc=-I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include inc=-I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include
all: gload gquery gserver gclient all: lib_antlr btree gload gquery gserver gclient
btree:
$(MAKE) -C KVstore
gload: $(objdir)gload.o $(objfile) gload: $(objdir)gload.o $(objfile)
g++ -o gload $(objdir)gload.o $(objfile) lib/libantlr.a $(CC) -g -o gload $(objdir)gload.o $(objfile) lib/libantlr.a
gquery: $(objdir)gquery.o $(objfile) gquery: $(objdir)gquery.o $(objfile)
g++ -o gquery $(objdir)gquery.o $(objfile) lib/libantlr.a $(CC) -g -o gquery $(objdir)gquery.o $(objfile) lib/libantlr.a
gserver: $(objdir)gserver.o $(objfile) gserver: $(objdir)gserver.o $(objfile)
g++ -o gserver $(objdir)gserver.o $(objfile) lib/libantlr.a $(CC) -g -o gserver $(objdir)gserver.o $(objfile) lib/libantlr.a
gclient: $(objdir)gclient.o $(objfile) gclient: $(objdir)gclient.o $(objfile)
g++ -o gclient $(objdir)gclient.o $(objfile) lib/libantlr.a $(CC) -g -o gclient $(objdir)gclient.o $(objfile) lib/libantlr.a
$(objdir)gload.o: main/gload.cpp $(objdir)gload.o: main/gload.cpp
g++ -c main/gload.cpp $(inc) -L./lib lib/libantlr.a -o $(objdir)gload.o $(CC) $(CFLAGS) main/gload.cpp $(inc) -L./lib lib/libantlr.a -o $(objdir)gload.o
$(objdir)gquery.o: main/gquery.cpp $(objdir)gquery.o: main/gquery.cpp
g++ -c main/gquery.cpp $(inc) -o $(objdir)gquery.o $(CC) $(CFLAGS) main/gquery.cpp $(inc) -o $(objdir)gquery.o
$(objdir)gserver.o: main/gserver.cpp $(objdir)gserver.o: main/gserver.cpp
g++ -c main/gserver.cpp $(inc) -o $(objdir)gserver.o $(CC) $(CFLAGS) main/gserver.cpp $(inc) -o $(objdir)gserver.o
$(objdir)gclient.o: main/gclient.cpp $(objdir)gclient.o: main/gclient.cpp
g++ -c main/gclient.cpp $(inc) -o $(objdir)gclient.o $(CC) $(CFLAGS) main/gclient.cpp $(inc) -o $(objdir)gclient.o
$(objdir)Bstr.o: Bstr/Bstr.cpp Bstr/Bstr.h $(objdir)Bstr.o: Bstr/Bstr.cpp Bstr/Bstr.h
g++ -c Bstr/Bstr.cpp $(inc) -o $(objdir)Bstr.o $(CC) $(CFLAGS) Bstr/Bstr.cpp $(inc) -o $(objdir)Bstr.o
$(objdir)Database.o: Database/Database.cpp Database/Database.h $(objdir)IDList.o $(objdir)ResultSet.o $(objdir)SPARQLquery.o \ $(objdir)Database.o: Database/Database.cpp Database/Database.h $(objdir)IDList.o $(objdir)ResultSet.o $(objdir)SPARQLquery.o \
$(objdir)BasicQuery.o \ $(objdir)BasicQuery.o \
$(objdir)Triple.o $(objdir)SigEntry.o $(objdir)KVstore.o $(objdir)VSTree.o $(objdir)DBparser.o $(objdir)util.o \ $(objdir)Triple.o $(objdir)SigEntry.o $(objdir)KVstore.o $(objdir)VSTree.o $(objdir)DBparser.o $(objdir)util.o \
$(objdir)RDFParser.o $(objdir)RDFParser.o
g++ -c Database/Database.cpp $(inc) -o $(objdir)Database.o $(CC) $(CFLAGS) Database/Database.cpp $(inc) -o $(objdir)Database.o
$(objdir)KVstore.o: KVstore/KVstore.cpp KVstore/KVstore.h $(objdir)Btree.o $(objdir)KVstore.o: KVstore/KVstore.cpp KVstore/KVstore.h
g++ -c KVstore/KVstore.cpp $(inc) -o $(objdir)KVstore.o $(CC) $(CFLAGS) KVstore/KVstore.cpp $(inc) -o $(objdir)KVstore.o
$(objdir)Btree.o: KVstore/Btree.cpp KVstore/Btree.h $(objdir)CBtreeFunc.o
g++ -c KVstore/Btree.cpp -o $(objdir)Btree.o
$(objdir)CBtreeFunc.o: KVstore/CBtreeFunc.cpp KVstore/CBtreeH.h
g++ -c KVstore/CBtreeFunc.cpp -o $(objdir)CBtreeFunc.o -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE
$(objdir)IDList.o: Query/IDList.cpp Query/IDList.h $(objdir)IDList.o: Query/IDList.cpp Query/IDList.h
g++ -c Query/IDList.cpp $(inc) -o $(objdir)IDList.o $(CC) $(CFLAGS) Query/IDList.cpp $(inc) -o $(objdir)IDList.o
$(objdir)SPARQLquery.o: Query/SPARQLquery.cpp Query/SPARQLquery.h $(objdir)BasicQuery.o $(objdir)SPARQLquery.o: Query/SPARQLquery.cpp Query/SPARQLquery.h $(objdir)BasicQuery.o
g++ -c Query/SPARQLquery.cpp $(inc) -o $(objdir)SPARQLquery.o $(CC) $(CFLAGS) Query/SPARQLquery.cpp $(inc) -o $(objdir)SPARQLquery.o
$(objdir)BasicQuery.o: Query/BasicQuery.cpp Query/BasicQuery.h $(objdir)Signature.o $(objdir)BasicQuery.o: Query/BasicQuery.cpp Query/BasicQuery.h $(objdir)Signature.o
g++ -c Query/BasicQuery.cpp $(inc) -o $(objdir)BasicQuery.o $(CC) $(CFLAGS) Query/BasicQuery.cpp $(inc) -o $(objdir)BasicQuery.o
$(objdir)ResultSet.o: Query/ResultSet.cpp Query/ResultSet.h $(objdir)ResultSet.o: Query/ResultSet.cpp Query/ResultSet.h
g++ -c Query/ResultSet.cpp $(inc) -o $(objdir)ResultSet.o $(CC) $(CFLAGS) Query/ResultSet.cpp $(inc) -o $(objdir)ResultSet.o
$(objdir)SigEntry.o: Signature/SigEntry.cpp Signature/SigEntry.h $(objdir)Signature.o $(objdir)SigEntry.o: Signature/SigEntry.cpp Signature/SigEntry.h $(objdir)Signature.o
g++ -c Signature/SigEntry.cpp $(inc) -o $(objdir)SigEntry.o $(CC) $(CFLAGS) Signature/SigEntry.cpp $(inc) -o $(objdir)SigEntry.o
$(objdir)Signature.o: Signature/Signature.cpp Signature/Signature.h $(objdir)Signature.o: Signature/Signature.cpp Signature/Signature.h
g++ -c Signature/Signature.cpp $(inc) -o $(objdir)Signature.o $(CC) $(CFLAGS) Signature/Signature.cpp $(inc) -o $(objdir)Signature.o
$(objdir)Triple.o: Triple/Triple.cpp Triple/Triple.h $(objdir)Triple.o: Triple/Triple.cpp Triple/Triple.h
g++ -c Triple/Triple.cpp $(inc) -o $(objdir)Triple.o $(CC) $(CFLAGS) Triple/Triple.cpp $(inc) -o $(objdir)Triple.o
$(objdir)util.o: util/util.cpp util/util.h $(objdir)util.o: util/util.cpp util/util.h
g++ -c util/util.cpp $(inc) -o $(objdir)util.o $(CC) $(CFLAGS) util/util.cpp $(inc) -o $(objdir)util.o
$(objdir)VSTree.o: VSTree/VSTree.cpp VSTree/VSTree.h $(objdir)EntryBuffer.o $(objdir)LRUCache.o $(objdir)VNode.o $(objdir)VSTree.o: VSTree/VSTree.cpp VSTree/VSTree.h $(objdir)EntryBuffer.o $(objdir)LRUCache.o $(objdir)VNode.o
g++ -c VSTree/VSTree.cpp $(inc) -o $(objdir)VSTree.o $(CC) $(CFLAGS) VSTree/VSTree.cpp $(inc) -o $(objdir)VSTree.o
$(objdir)EntryBuffer.o: VSTree/EntryBuffer.cpp VSTree/EntryBuffer.h Signature/SigEntry.h $(objdir)EntryBuffer.o: VSTree/EntryBuffer.cpp VSTree/EntryBuffer.h Signature/SigEntry.h
g++ -c VSTree/EntryBuffer.cpp $(inc) -o $(objdir)EntryBuffer.o $(CC) $(CFLAGS) VSTree/EntryBuffer.cpp $(inc) -o $(objdir)EntryBuffer.o
$(objdir)LRUCache.o: VSTree/LRUCache.cpp VSTree/LRUCache.h VSTree/VNode.h $(objdir)LRUCache.o: VSTree/LRUCache.cpp VSTree/LRUCache.h VSTree/VNode.h
g++ -c VSTree/LRUCache.cpp $(inc) -o $(objdir)LRUCache.o $(CC) $(CFLAGS) VSTree/LRUCache.cpp $(inc) -o $(objdir)LRUCache.o
$(objdir)VNode.o: VSTree/VNode.cpp VSTree/VNode.h $(objdir)VNode.o: VSTree/VNode.cpp VSTree/VNode.h
g++ -c VSTree/VNode.cpp $(inc) -o $(objdir)VNode.o $(CC) $(CFLAGS) VSTree/VNode.cpp $(inc) -o $(objdir)VNode.o
$(objdir)DBparser.o: Parser/DBparser.cpp Parser/DBparser.h $(objdir)SparqlParser.o $(objdir)SparqlLexer.o $(objdir)Triple.o $(objdir)DBparser.o: Parser/DBparser.cpp Parser/DBparser.h $(objdir)SparqlParser.o $(objdir)SparqlLexer.o $(objdir)Triple.o
g++ -c Parser/DBparser.cpp $(inc) -o $(objdir)DBparser.o $(CC) $(CFLAGS) Parser/DBparser.cpp $(inc) -o $(objdir)DBparser.o
$(objdir)SparqlParser.o: Parser/SparqlParser.c Parser/SparqlParser.h $(objdir)SparqlParser.o: Parser/SparqlParser.c Parser/SparqlParser.h
gcc -c Parser/SparqlParser.c $(inc) -o $(objdir)SparqlParser.o gcc $(CFLAGS) Parser/SparqlParser.c $(inc) -o $(objdir)SparqlParser.o
$(objdir)SparqlLexer.o: Parser/SparqlLexer.c Parser/SparqlLexer.h $(objdir)SparqlLexer.o: Parser/SparqlLexer.c Parser/SparqlLexer.h
gcc -c Parser/SparqlLexer.c $(inc) -o $(objdir)SparqlLexer.o gcc $(CFLAGS) Parser/SparqlLexer.c $(inc) -o $(objdir)SparqlLexer.o
$(objdir)TurtleParser.o: Parser/TurtleParser.cpp Parser/TurtleParser.h Parser/Type.h $(objdir)TurtleParser.o: Parser/TurtleParser.cpp Parser/TurtleParser.h Parser/Type.h
gcc -c Parser/TurtleParser.cpp $(inc) -o $(objdir)TurtleParser.o gcc $(CFLAGS) Parser/TurtleParser.cpp $(inc) -o $(objdir)TurtleParser.o
$(objdir)RDFParser.o: Parser/RDFParser.cpp Parser/RDFParser.h $(objdir)TurtleParser.o $(objdir)Triple.o $(objdir)RDFParser.o: Parser/RDFParser.cpp Parser/RDFParser.h $(objdir)TurtleParser.o $(objdir)Triple.o
gcc -c Parser/RDFParser.cpp $(inc) -o $(objdir)RDFParser.o gcc $(CFLAGS) Parser/RDFParser.cpp $(inc) -o $(objdir)RDFParser.o
$(objdir)Operation.o: Server/Operation.cpp Server/Operation.h $(objdir)Operation.o: Server/Operation.cpp Server/Operation.h
g++ -c Server/Operation.cpp $(inc) -o $(objdir)Operation.o $(CC) $(CFLAGS) Server/Operation.cpp $(inc) -o $(objdir)Operation.o
$(objdir)Socket.o: Server/Socket.cpp Server/Socket.h $(objdir)Socket.o: Server/Socket.cpp Server/Socket.h
g++ -c Server/Socket.cpp $(inc) -o $(objdir)Socket.o $(CC) $(CFLAGS) Server/Socket.cpp $(inc) -o $(objdir)Socket.o
$(objdir)Server.o: Server/Server.cpp Server/Server.h $(objdir)Socket.o $(objdir)Database.o $(objdir)Operation.o $(objdir)Server.o: Server/Server.cpp Server/Server.h $(objdir)Socket.o $(objdir)Database.o $(objdir)Operation.o
g++ -c Server/Server.cpp $(inc) -o $(objdir)Server.o $(CC) $(CFLAGS) Server/Server.cpp $(inc) -o $(objdir)Server.o
$(objdir)Client.o: Server/Client.cpp Server/Client.h $(objdir)Socket.o $(objdir)Client.o: Server/Client.cpp Server/Client.h $(objdir)Socket.o
g++ -c Server/Client.cpp $(inc) -o $(objdir)Client.o $(CC) $(CFLAGS) Server/Client.cpp $(inc) -o $(objdir)Client.o
lib_antlr: lib_antlr:
rm -rf tools/libantlr3c-3.4/ rm -rf tools/libantlr3c-3.4/
@ -107,5 +108,6 @@ lib_antlr:
ar -crv lib/libantlr.a tools/libantlr3c-3.4/*.o ar -crv lib/libantlr.a tools/libantlr3c-3.4/*.o
clean: clean:
$(MAKE) -C KVstore clean
rm -rf gload gquery gserver gclient $(objdir)/*.o rm -rf gload gquery gserver gclient $(objdir)/*.o