opensci
/
gStore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix bugs in api and gclient; 

fix bugs in gconsole, pre_var and vstree;
support selected pre vars and parallel edges;
update both en and ch docs
This commit is contained in:
bookug 2017-02-17 17:09:47 +08:00
parent f25ebcb123
commit 74598bf66e
52 changed files with 2228 additions and 823 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

261
Database/Database.cpp
View File

@ -145,7 +145,7 @@ Database::readIDinfo()
fp = fopen(this->free_id_file_entity.c_str(), "r");
if (fp == NULL)
{
cerr << "read entity id info error" << endl;
cout << "read entity id info error" << endl;
return;
}
//QUERY:this will reverse the original order, if change?
@ -181,7 +181,7 @@ Database::readIDinfo()
fp = fopen(this->free_id_file_literal.c_str(), "r");
if (fp == NULL)
{
cerr << "read literal id info error" << endl;
cout << "read literal id info error" << endl;
return;
}
fread(&(this->limitID_literal), sizeof(int), 1, fp);
@ -198,7 +198,7 @@ Database::readIDinfo()
fp = fopen(this->free_id_file_predicate.c_str(), "r");
if (fp == NULL)
{
cerr << "read predicate id info error" << endl;
cout << "read predicate id info error" << endl;
return;
}
fread(&(this->limitID_predicate), sizeof(int), 1, fp);
@ -223,7 +223,7 @@ Database::writeIDinfo()
fp = fopen(this->free_id_file_entity.c_str(), "w+");
if (fp == NULL)
{
cerr << "write entity id info error" << endl;
cout << "write entity id info error" << endl;
return;
}
fwrite(&(this->limitID_entity), sizeof(int), 1, fp);
@ -245,7 +245,7 @@ Database::writeIDinfo()
fp = fopen(this->free_id_file_literal.c_str(), "w+");
if (fp == NULL)
{
cerr << "write literal id info error" << endl;
cout << "write literal id info error" << endl;
return;
}
fwrite(&(this->limitID_literal), sizeof(int), 1, fp);
@ -263,7 +263,7 @@ Database::writeIDinfo()
fp = fopen(this->free_id_file_predicate.c_str(), "w+");
if (fp == NULL)
{
cerr << "write predicate id info error" << endl;
cout << "write predicate id info error" << endl;
return;
}
fwrite(&(this->limitID_predicate), sizeof(int), 1, fp);
@ -289,7 +289,7 @@ Database::allocEntityID()
t = this->limitID_entity++;
if (this->limitID_entity >= Util::LITERAL_FIRST_ID)
{
cerr << "fail to alloc id for entity" << endl;
cout << "fail to alloc id for entity" << endl;
return -1;
}
}
@ -330,7 +330,7 @@ Database::allocLiteralID()
t = this->limitID_literal++;
if (this->limitID_literal >= Util::LITERAL_FIRST_ID)
{
cerr << "fail to alloc id for literal" << endl;
cout << "fail to alloc id for literal" << endl;
return -1;
}
}
@ -371,7 +371,7 @@ Database::allocPredicateID()
t = this->limitID_predicate++;
if (this->limitID_predicate >= Util::LITERAL_FIRST_ID)
{
cerr << "fail to alloc id for predicate" << endl;
cout << "fail to alloc id for predicate" << endl;
return -1;
}
}
@ -408,6 +408,7 @@ Database::release(FILE* fp0)
{
fprintf(fp0, "begin to delete DB!\n");
fflush(fp0);
this->vstree->saveTree();
delete this->vstree;
fprintf(fp0, "ok to delete vstree!\n");
fflush(fp0);
@ -445,6 +446,8 @@ Database::setPreMap()
t = this->kvstore->getPredicateDegree(i);
}
this->pre2num[i] = t;
//NOTICE:only when pre2num[i]>0 then i is a valid predicate id
if (t > 0)
{
valid++;
@ -532,14 +535,14 @@ Database::load()
bool flag = (this->vstree)->loadTree();
if (!flag)
{
cerr << "load tree error. @Database::load()" << endl;
cout << "load tree error. @Database::load()" << endl;
return false;
}
flag = this->loadDBInfoFile();
if (!flag)
{
cerr << "load database info error. @Database::load()" << endl;
cout << "load database info error. @Database::load()" << endl;
return false;
}
@ -572,6 +575,7 @@ Database::unload()
delete this->entity_buffer;
delete this->literal_buffer;
this->vstree->saveTree();
delete this->vstree;
this->vstree = NULL;
delete this->kvstore;
@ -579,6 +583,7 @@ Database::unload()
delete this->stringindex;
this->stringindex = NULL;
this->saveDBInfoFile();
this->writeIDinfo();
this->initIDinfo();
return true;
@ -590,10 +595,10 @@ Database::getName()
return this->name;
}
bool
int
Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
{
GeneralEvaluation general_evaluation(this->vstree, this->kvstore, this->stringindex, this->pre2num);
GeneralEvaluation general_evaluation(this->vstree, this->kvstore, this->stringindex, this->pre2num, this->limitID_predicate, this->limitID_literal);
long tv_begin = Util::get_cur_time();
@ -602,21 +607,32 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
long tv_parse = Util::get_cur_time();
cout << "after Parsing, used " << (tv_parse - tv_begin) << "ms." << endl;
//TODO:output all results in JSON format, and transformed into string to client
//for select, -100 by default, -101 means error
//for update, non-negative means true(and the num is updated triples num), -1 means error
int success_num = -100;
//Query
if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Not_Update)
{
general_evaluation.doQuery();
bool query_ret = general_evaluation.doQuery();
if(!query_ret)
{
success_num = -101;
}
long tv_bfget = Util::get_cur_time();
general_evaluation.getFinalResult(_result_set);
long tv_afget = Util::get_cur_time();
cout << "after getFinalResult, used " << (tv_afget - tv_bfget) << "ms." << endl;
general_evaluation.setNeedOutputAnswer();
if(_fp != NULL)
general_evaluation.setNeedOutputAnswer();
}
//Update
else
{
success_num = 0;
TripleWithObjType *update_triple = NULL;
int update_triple_num = 0;
@ -630,18 +646,24 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
for (int i = 0; i < (int)update_pattern.patterns.size(); i++)
{
TripleWithObjType::ObjectType object_type = TripleWithObjType::None;
if (update_pattern.patterns[i].object.value[0] == '<')
object_type = TripleWithObjType::Entity;
else
object_type = TripleWithObjType::Literal;
update_triple[i] = TripleWithObjType(update_pattern.patterns[i].subject.value,
update_pattern.patterns[i].predicate.value,
update_pattern.patterns[i].object.value);
update_pattern.patterns[i].object.value, object_type);
}
if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Insert_Data)
{
insert(update_triple, update_triple_num);
success_num = insert(update_triple, update_triple_num);
}
else if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Data)
{
remove(update_triple, update_triple_num);
success_num = remove(update_triple, update_triple_num);
}
}
else if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Where || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Insert_Clause ||
@ -653,12 +675,12 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Where || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Clause || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Modify_Clause)
{
general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getDeletePatterns(), update_triple, update_triple_num);
remove(update_triple, update_triple_num);
success_num = remove(update_triple, update_triple_num);
}
if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Insert_Clause || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Modify_Clause)
{
general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getInsertPatterns(), update_triple, update_triple_num);
insert(update_triple, update_triple_num);
success_num = insert(update_triple, update_triple_num);
}
}
@ -673,27 +695,15 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
{
cout << "There has answer: " << _result_set.ansNum << endl;
cout << "final result is : " << endl;
//if (!_result_set.checkUseStream())
//{
//cout << _result_set.to_str() << endl;
//}
//else
//{
_result_set.output(_fp);
//cout<<endl; //empty the buffer;print an empty line
fprintf(_fp, "\n");
fflush(_fp); //to empty the output buffer in C (fflush(stdin) not work in GCC)
//}
//_result_set.output(_fp);
//fprintf(_fp, "\n");
//fflush(_fp); //to empty the output buffer in C (fflush(stdin) not work in GCC)
_result_set.output(_fp);
fprintf(_fp, "\n");
fflush(_fp); //to empty the output buffer in C (fflush(stdin) not work in GCC)
}
#ifdef DEBUG_PRECISE
fprintf(stderr, "the query function exits!\n");
#ifdef DEBUG
cout<<"query success_num: "<<success_num<<endl;
#endif
return true;
return success_num;
}
//NOTICE+QUERY:to save memory for large cases, we can consider building one tree at a time(then release)
@ -767,8 +777,8 @@ Database::build(const string& _rdf_file)
cout << "finish build VS-Tree." << endl;
//this->vstree->saveTree();
delete this->vstree;
this->vstree = NULL;
//delete this->vstree;
//this->vstree = NULL;
//sync();
//cout << "sync vstree" << endl;
@ -807,7 +817,7 @@ Database::saveDBInfoFile()
if (filePtr == NULL)
{
cerr << "error, can not create db info file. @Database::saveDBInfoFile" << endl;
cout << "error, can not create db info file. @Database::saveDBInfoFile" << endl;
return false;
}
@ -836,7 +846,7 @@ Database::loadDBInfoFile()
if (filePtr == NULL)
{
cerr << "error, can not open db info file. @Database::loadDBInfoFile" << endl;
cout << "error, can not open db info file. @Database::loadDBInfoFile" << endl;
return false;
}
@ -995,9 +1005,9 @@ bool
Database::encodeRDF_new(const string _rdf_file)
{
#ifdef DEBUG
//cerr<< "now to log!!!" << endl;
//cout<< "now to log!!!" << endl;
Util::logging("In encodeRDF_new");
//cerr<< "end log!!!" << endl;
//cout<< "end log!!!" << endl;
#endif
int** _p_id_tuples = NULL;
int _id_tuples_max = 0;
@ -1080,13 +1090,13 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file, int**& _
}
//Util::logging("finish initial sub2id_pre2id_obj2id");
cerr << "finish initial sub2id_pre2id_obj2id" << endl;
cout << "finish initial sub2id_pre2id_obj2id" << endl;
//BETTER?:close the stdio buffer sync??
ifstream _fin(_rdf_file.c_str());
if (!_fin)
{
cerr << "sub2id&pre2id&obj2id: Fail to open : " << _rdf_file << endl;
cout << "sub2id&pre2id&obj2id: Fail to open : " << _rdf_file << endl;
//exit(0);
return false;
}
@ -1095,7 +1105,7 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file, int**& _
ofstream _six_tuples_fout(_six_tuples_file.c_str());
if (!_six_tuples_fout)
{
cerr << "sub2id&pre2id&obj2id: Fail to open: " << _six_tuples_file << endl;
cout << "sub2id&pre2id&obj2id: Fail to open: " << _six_tuples_file << endl;
//exit(0);
return false;
}
@ -1287,7 +1297,7 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file, int**& _
string _sig_binary_file = this->getSignatureBFile();
FILE* _sig_fp = fopen(_sig_binary_file.c_str(), "wb");
if (_sig_fp == NULL) {
cerr << "Failed to open : " << _sig_binary_file << endl;
cout << "Failed to open : " << _sig_binary_file << endl;
}
//EntityBitSet _all_bitset;
@ -1321,7 +1331,7 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file, int**& _
return true;
}
int
bool
Database::insertTriple(const TripleWithObjType& _triple, vector<int>* _vertices, vector<int>* _predicates)
{
//cout<<endl<<"the new triple is:"<<endl;
@ -1449,7 +1459,7 @@ Database::insertTriple(const TripleWithObjType& _triple, vector<int>* _vertices,
if (_triple_exist)
{
cout << "this triple already exist" << endl;
return 0;
return false;
}
else
{
@ -1485,7 +1495,7 @@ Database::insertTriple(const TripleWithObjType& _triple, vector<int>* _vertices,
//if new entity then insert it, else update it.
if (_is_new_sub)
{
//cout<<"to insert: "<<_sub_id<<" "<<this->kvstore->getEntityByID(_sub_id)<<endl;
cout<<"to insert: "<<_sub_id<<" "<<this->kvstore->getEntityByID(_sub_id)<<endl;
SigEntry _sig(_sub_id, _sub_entity_bitset);
(this->vstree)->insertEntry(_sig);
}
@ -1524,7 +1534,8 @@ Database::insertTriple(const TripleWithObjType& _triple, vector<int>* _vertices,
cout << "update vs_store, used " << (tv_vs_store_end - tv_kv_store_end) << "ms." << endl;
}
return updateLen;
return true;
//return updateLen;
}
bool
@ -1567,8 +1578,8 @@ Database::removeTriple(const TripleWithObjType& _triple, vector<int>* _vertices,
//if subject become an isolated point, remove its corresponding entry
if (sub_degree == 0)
{
//cout<<"to remove entry for sub"<<endl;
//cout<<_sub_id << " "<<this->kvstore->getEntityByID(_sub_id)<<endl;
cout<<"to remove entry for sub"<<endl;
cout<<_sub_id << " "<<this->kvstore->getEntityByID(_sub_id)<<endl;
this->kvstore->subEntityByID(_sub_id);
this->kvstore->subIDByEntity(_triple.subject);
(this->vstree)->removeEntry(_sub_id);
@ -1680,10 +1691,12 @@ Database::insert(std::string _rdf_file)
long tv_load = Util::get_cur_time();
int success_num = 0;
ifstream _fin(_rdf_file.c_str());
if (!_fin)
{
cerr << "fail to open : " << _rdf_file << ".@insert_test" << endl;
cout << "fail to open : " << _rdf_file << ".@insert_test" << endl;
//exit(0);
return false;
}
@ -1717,7 +1730,7 @@ Database::insert(std::string _rdf_file)
}
//Process the Triple one by one
this->insert(triple_array, parse_triple_num);
success_num += this->insert(triple_array, parse_triple_num);
//some maybe invalid or duplicate
//triple_num += parse_triple_num;
}
@ -1726,18 +1739,19 @@ Database::insert(std::string _rdf_file)
long tv_insert = Util::get_cur_time();
cout << "after insert, used " << (tv_insert - tv_load) << "ms." << endl;
//BETTER:update kvstore and vstree separately, to lower the memory cost
flag = this->vstree->saveTree();
if (!flag)
{
return false;
}
flag = this->saveDBInfoFile();
if (!flag)
{
return false;
}
//flag = this->vstree->saveTree();
//if (!flag)
//{
//return false;
//}
//flag = this->saveDBInfoFile();
//if (!flag)
//{
//return false;
//}
cout << "insert rdf triples done." << endl;
cout<<"inserted triples num: "<<success_num<<endl;
//int* list = NULL;
//int len = 0;
@ -1772,11 +1786,12 @@ Database::remove(std::string _rdf_file)
cout << "finish loading" << endl;
long tv_load = Util::get_cur_time();
int success_num = 0;
ifstream _fin(_rdf_file.c_str());
if (!_fin)
{
cerr << "fail to open : " << _rdf_file << ".@remove_test" << endl;
cout << "fail to open : " << _rdf_file << ".@remove_test" << endl;
return false;
}
@ -1805,7 +1820,7 @@ Database::remove(std::string _rdf_file)
break;
}
this->remove(triple_array, parse_triple_num);
success_num += this->remove(triple_array, parse_triple_num);
//some maybe invalid or duplicate
//triple_num -= parse_triple_num;
}
@ -1817,31 +1832,33 @@ Database::remove(std::string _rdf_file)
long tv_remove = Util::get_cur_time();
cout << "after remove, used " << (tv_remove - tv_load) << "ms." << endl;
flag = this->vstree->saveTree();
if (!flag)
{
return false;
}
flag = this->saveDBInfoFile();
if (!flag)
{
return false;
}
//flag = this->vstree->saveTree();
//if (!flag)
//{
//return false;
//}
//flag = this->saveDBInfoFile();
//if (!flag)
//{
//return false;
//}
cout << "remove rdf triples done." << endl;
cout<<"removed triples num: "<<success_num<<endl;
return true;
}
bool
int
Database::insert(const TripleWithObjType* _triples, int _triple_num)
{
vector<int> vertices, predicates;
int valid_num = 0;
#ifdef USE_GROUP_INSERT
//NOTICE:this is called by insert(file) or query()(but can not be too large),
//assume that db is loaded already
int** id_tuples = new int*[_triple_num];
int valid_num = 0;
int i = 0;
//for(i = 0; i < _triple_num; ++i)
//{
@ -1939,10 +1956,14 @@ Database::insert(const TripleWithObjType* _triples, int _triple_num)
}
if (triple_exist)
{
#ifdef DEBUG
cout << "this triple exist" << endl;
#endif
continue;
}
#ifdef DEBUG
cout << "this triple not exist" << endl;
#endif
id_tuples[valid_num] = new int[3];
id_tuples[valid_num][0] = subid;
@ -2011,9 +2032,11 @@ Database::insert(const TripleWithObjType* _triples, int _triple_num)
}
}
#ifdef DEBUG
cout << "old sigmap size: " << old_sigmap.size() << endl;
cout << "new sigmap size: " << new_sigmap.size() << endl;
cout << "valid num: " << valid_num << endl;
#endif
//NOTICE:need to sort and remove duplicates, update the valid num
//Notice that duplicates in a group can csuse problem
@ -2027,7 +2050,9 @@ Database::insert(const TripleWithObjType* _triples, int _triple_num)
//
//spo cmp: s2p s2o s2po sp2o
{
#ifdef DEBUG
cout << "INSRET PROCESS: to spo cmp and update" << endl;
#endif
qsort(id_tuples, valid_num, sizeof(int*), KVstore::_spo_cmp);
//To remove duplicates
@ -2080,7 +2105,9 @@ Database::insert(const TripleWithObjType* _triples, int _triple_num)
if (_sub_pre_change)
{
#ifdef DEBUG
cout << "update sp2o: " << _sub_id << " " << _pre_id << " " << oidlist_sp.size() << endl;
#endif
cout << this->kvstore->getEntityByID(_sub_id) << endl;
cout << this->kvstore->getPredicateByID(_pre_id) << endl;
//this->kvstore->updateInsert_sp2o(_sub_id, _pre_id, oidlist_sp);
@ -2089,26 +2116,36 @@ Database::insert(const TripleWithObjType* _triples, int _triple_num)
if (_sub_change)
{
#ifdef DEBUG
cout << "update s2p: " << _sub_id << " " << pidlist_s.size() << endl;
#endif
//this->kvstore->updateInsert_s2p(_sub_id, pidlist_s);
pidlist_s.clear();
#ifdef DEBUG
cout << "update s2po: " << _sub_id << " " << pidoidlist_s.size() << endl;
#endif
this->kvstore->updateInsert_s2values(_sub_id, pidoidlist_s);
pidoidlist_s.clear();
#ifdef DEBUG
cout << "update s2o: " << _sub_id << " " << oidlist_s.size() << endl;
#endif
sort(oidlist_s.begin(), oidlist_s.end());
//this->kvstore->updateInsert_s2o(_sub_id, oidlist_s);
oidlist_s.clear();
}
}
cerr << "INSERT PROCESS: OUT s2po..." << endl;
#ifdef DEBUG
cout << "INSERT PROCESS: OUT s2po..." << endl;
#endif
}
//ops cmp: o2p o2s o2ps op2s
{
#ifdef DEBUG
cout << "INSRET PROCESS: to ops cmp and update" << endl;
#endif
qsort(id_tuples, valid_num, sizeof(int**), KVstore::_ops_cmp);
vector<int> sidlist_o;
vector<int> sidlist_op;
@ -2138,33 +2175,45 @@ Database::insert(const TripleWithObjType* _triples, int _triple_num)
if (_obj_pre_change)
{
#ifdef DEBUG
cout << "update op2s: " << _obj_id << " " << _pre_id << " " << sidlist_op.size() << endl;
#endif
//this->kvstore->updateInsert_op2s(_obj_id, _pre_id, sidlist_op);
sidlist_op.clear();
}
if (_obj_change)
{
#ifdef DEBUG
cout << "update o2s: " << _obj_id << " " << sidlist_o.size() << endl;
#endif
sort(sidlist_o.begin(), sidlist_o.end());
//this->kvstore->updateInsert_o2s(_obj_id, sidlist_o);
sidlist_o.clear();
#ifdef DEBUG
cout << "update o2ps: " << _obj_id << " " << pidsidlist_o.size() << endl;
#endif
this->kvstore->updateInsert_o2values(_obj_id, pidsidlist_o);
pidsidlist_o.clear();
#ifdef DEBUG
cout << "update o2p: " << _obj_id << " " << pidlist_o.size() << endl;
#endif
//this->kvstore->updateInsert_o2p(_obj_id, pidlist_o);
pidlist_o.clear();
}
}
cerr << "INSERT PROCESS: OUT o2ps..." << endl;
#ifdef DEBUG
cout << "INSERT PROCESS: OUT o2ps..." << endl;
#endif
}
//pso cmp: p2s p2o p2so
{
#ifdef DEBUG
cout << "INSRET PROCESS: to pso cmp and update" << endl;
#endif
qsort(id_tuples, valid_num, sizeof(int*), KVstore::_pso_cmp);
vector<int> sidlist_p;
vector<int> oidlist_p;
@ -2192,21 +2241,29 @@ Database::insert(const TripleWithObjType* _triples, int _triple_num)
if (_pre_change)
{
#ifdef DEBUG
cout << "update p2s: " << _pre_id << " " << sidlist_p.size() << endl;
#endif
//this->kvstore->updateInsert_p2s(_pre_id, sidlist_p);
sidlist_p.clear();
#ifdef DEBUG
cout << "update p2o: " << _pre_id << " " << oidlist_p.size() << endl;
#endif
sort(oidlist_p.begin(), oidlist_p.end());
//this->kvstore->updateInsert_p2o(_pre_id, oidlist_p);
oidlist_p.clear();
#ifdef DEBUG
cout << "update p2so: " << _pre_id << " " << sidoidlist_p.size() << endl;
#endif
this->kvstore->updateInsert_p2values(_pre_id, sidoidlist_p);
sidoidlist_p.clear();
}
}
cerr << "INSERT PROCESS: OUT p2so..." << endl;
#ifdef DEBUG
cout << "INSERT PROCESS: OUT p2so..." << endl;
#endif
}
@ -2230,7 +2287,11 @@ Database::insert(const TripleWithObjType* _triples, int _triple_num)
//Callers should save the vstree(node and info) after calling this function
for (int i = 0; i < _triple_num; ++i)
{
this->insertTriple(_triples[i], &vertices, &predicates);
bool ret = this->insertTriple(_triples[i], &vertices, &predicates);
if(ret)
{
valid_num++;
}
}
#endif
@ -2238,19 +2299,19 @@ Database::insert(const TripleWithObjType* _triples, int _triple_num)
this->stringindex->change(vertices, *this->kvstore, true);
this->stringindex->change(predicates, *this->kvstore, false);
return true;
return valid_num;
}
bool
int
Database::remove(const TripleWithObjType* _triples, int _triple_num)
{
vector<int> vertices, predicates;
int valid_num = 0;
#ifdef USE_GROUP_DELETE
//NOTICE:this is called by remove(file) or query()(but can not be too large),
//assume that db is loaded already
int** id_tuples = new int*[_triple_num];
int valid_num = 0;
int i = 0;
//for(i = 0; i < _triple_num; ++i)
//{
@ -2311,7 +2372,9 @@ Database::remove(const TripleWithObjType* _triples, int _triple_num)
//
//spo cmp: s2p s2o s2po sp2o
{
#ifdef DEBUG
cout << "INSRET PROCESS: to spo cmp and update" << endl;
#endif
qsort(id_tuples, valid_num, sizeof(int*), KVstore::_spo_cmp);
vector<int> oidlist_s;
vector<int> pidlist_s;
@ -2381,11 +2444,15 @@ Database::remove(const TripleWithObjType* _triples, int _triple_num)
}
}
cerr << "INSERT PROCESS: OUT s2po..." << endl;
#ifdef DEBUG
cout << "INSERT PROCESS: OUT s2po..." << endl;
#endif
}
//ops cmp: o2p o2s o2ps op2s
{
#ifdef DEBUG
cout << "INSRET PROCESS: to ops cmp and update" << endl;
#endif
qsort(id_tuples, valid_num, sizeof(int**), KVstore::_ops_cmp);
vector<int> sidlist_o;
vector<int> sidlist_op;
@ -2476,11 +2543,15 @@ Database::remove(const TripleWithObjType* _triples, int _triple_num)
}
}
cerr << "INSERT PROCESS: OUT o2ps..." << endl;
#ifdef DEBUG
cout << "INSERT PROCESS: OUT o2ps..." << endl;
#endif
}
//pso cmp: p2s p2o p2so
{
#ifdef DEBUG
cout << "INSRET PROCESS: to pso cmp and update" << endl;
#endif
qsort(id_tuples, valid_num, sizeof(int*), KVstore::_pso_cmp);
vector<int> sidlist_p;
vector<int> oidlist_p;
@ -2530,7 +2601,9 @@ Database::remove(const TripleWithObjType* _triples, int _triple_num)
}
}
}
cerr << "INSERT PROCESS: OUT p2so..." << endl;
#ifdef DEBUG
cout << "INSERT PROCESS: OUT p2so..." << endl;
#endif
}
@ -2544,7 +2617,11 @@ Database::remove(const TripleWithObjType* _triples, int _triple_num)
//Callers should save the vstree(node and info) after calling this function
for (int i = 0; i < _triple_num; ++i)
{
this->removeTriple(_triples[i], &vertices, &predicates);
bool ret = this->removeTriple(_triples[i], &vertices, &predicates);
if(ret)
{
valid_num++;
}
}
#endif
@ -2552,7 +2629,7 @@ Database::remove(const TripleWithObjType* _triples, int _triple_num)
this->stringindex->disable(vertices, true);
this->stringindex->disable(predicates, false);
return true;
return valid_num;
}
bool

9
Database/Database.h
View File

@ -47,7 +47,7 @@ public:
bool load();
bool unload();
bool query(const string _query, ResultSet& _result_set, FILE* _fp = stdout);
int query(const string _query, ResultSet& _result_set, FILE* _fp = stdout);
//1. if subject of _triple doesn't exist,
//then assign a new subid, and insert a new SigEntry
@ -96,6 +96,7 @@ private:
//pre2num mapping
TNUM* pre2num;
//valid: check from minNumPID to maxNumPID
int maxNumPID, minNumPID;
void setPreMap();
//string buffer
@ -172,12 +173,12 @@ private:
//insert and delete, notice that modify is not needed here
//we can read from file or use sparql syntax
int insertTriple(const TripleWithObjType& _triple, vector<int>* _vertices = NULL, vector<int>* _predicates = NULL);
bool insertTriple(const TripleWithObjType& _triple, vector<int>* _vertices = NULL, vector<int>* _predicates = NULL);
bool removeTriple(const TripleWithObjType& _triple, vector<int>* _vertices = NULL, vector<int>* _predicates = NULL);
//NOTICE:one by one is too costly, sort and insert/delete at a time will be better
bool insert(const TripleWithObjType* _triples, int _triple_num);
int insert(const TripleWithObjType* _triples, int _triple_num);
//bool insert(const vector<TripleWithObjType>& _triples, vector<int>& _vertices, vector<int>& _predicates);
bool remove(const TripleWithObjType* _triples, int _triple_num);
int remove(const TripleWithObjType* _triples, int _triple_num);
//bool remove(const vector<TripleWithObjType>& _triples, vector<int>& _vertices, vector<int>& _predicates);
bool sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file, int**& _p_id_tuples, int & _id_tuples_max);

549
Database/Join.cpp
View File

File diff suppressed because it is too large Load Diff

7
Database/Join.h
View File

@ -48,6 +48,8 @@ private:
BasicQuery* basic_query;
KVstore* kvstore;
TNUM* pre2num;
int limitID_predicate;
int limitID_literal;
//used by score_node for parameters
static const unsigned PARAM_DEGREE = 1;
static const unsigned PARAM_SIZE = 1000000;
@ -118,7 +120,8 @@ private:
//BETTER?:change these params to members in class
void acquire_all_id_lists(IdLists& _id_lists, IdListsLen& _id_lists_len, IDList& _can_list, vector<int>& _edges, int _id, int _can_list_size);
bool join_two(vector<int>& _edges, IDList& _can_list, int _can_list_size, int _id, bool _is_literal);
void update_answer_list(IDList*& valid_ans_list, IDList& _can_list, int* id_list, int id_list_len, bool _is_literal);
bool join_two(vector< vector<int> >& _edges, IDList& _can_list, int _can_list_size, int _id, bool _is_literal);
bool multi_join();
//NOTICE:this is only used to join a BasicQuery
@ -126,7 +129,7 @@ private:
public:
Join();
Join(KVstore* _kvstore, TNUM* _pre2num);
Join(KVstore* _kvstore, TNUM* _pre2num, int _limitID_predicate, int _limitID_literal);
//these functions can be called by Database
bool join_sparql(SPARQLquery& _sparql_query);
bool join_basic(BasicQuery* _basic_query);

276
Database/Strategy.cpp
View File

@ -18,12 +18,14 @@ Strategy::Strategy()
//this->prepare_handler();
}
Strategy::Strategy(KVstore* _kvstore, VSTree* _vstree, TNUM* _pre2num)
Strategy::Strategy(KVstore* _kvstore, VSTree* _vstree, TNUM* _pre2num, int _limitID_predicate, int _limitID_literal)
{
this->method = 0;
this->kvstore = _kvstore;
this->vstree = _vstree;
this->pre2num = _pre2num;
this->limitID_predicate = _limitID_predicate;
this->limitID_literal = _limitID_literal;
//this->prepare_handler();
}
@ -40,7 +42,7 @@ Strategy::~Strategy()
//}
//NOTICE: 2-triple case ?s1 p1 c0 ?s2 p2 c0 is viewed as an unconnected graph
//however, this can be dealed due to several basicquery and linking
//however, this can be dealed due to several basic queries and linking
bool
Strategy::handle(SPARQLquery& _query, ResultFilter* _result_filter)
@ -54,84 +56,24 @@ Strategy::handle(SPARQLquery& _query, ResultFilter* _result_filter)
for (; iter != queryList.end(); iter++)
if ((**iter).getEncodeBasicQueryResult())
{
this->method = 0;
this->method = -1;
vector<int*>& result_list = (*iter)->getResultList();
//int select_var_num = (*iter)->getSelectVarNum();
int varNum = (*iter)->getVarNum(); //the num of vars needing to be joined
//the num of vars needing to be joined, i.e. selectVarNum if only one triple
int varNum = (*iter)->getVarNum();
//all variables(not including pre vars)
int total_num = (*iter)->getTotalVarNum();
int pre_varNum = (*iter)->getPreVarNum();
int selected_pre_var_num = (*iter)->getSelectedPreVarNum();
int selected_var_num = (*iter)->getSelectVarNum();
//TODO:if need to return pre var?
if ((*iter)->getTripleNum() == 1 && pre_varNum == 1)
{
Triple triple = (*iter)->getTriple(0);
int* id_list = NULL;
int id_list_len = 0;
result_list.clear();
if (total_num == 2)
{
//TODO:consider special case, select ?s (?p) ?o where { ?s ?p ?o . }
//filter and join is too costly, should enum all predicates and use p2so
//maybe the selected vars are ?s (?p) or ?o (?p)
cerr << "not supported now!" << endl;
}
else if (total_num == 1)
{
//TODO:if just select s/o, use o2s/s2o
//if only p is selected, use s2p or o2p
//only if both s/o and p are selected, use s2po or o2ps
if(triple.subject[0] != '?') //constant
{
int sid = (this->kvstore)->getIDByEntity(triple.subject);
this->kvstore->getpreIDobjIDlistBysubID(sid, id_list, id_list_len);
}
else if(triple.object[0] != '?') //constant
{
int oid = (this->kvstore)->getIDByEntity(triple.object);
if (oid == -1)
{
oid = (this->kvstore)->getIDByLiteral(triple.object);
}
this->kvstore->getpreIDsubIDlistByobjID(oid, id_list, id_list_len);
}
//always place s/o before p in result list
for (int i = 0; i < id_list_len; i += 2)
{
int* record = new int[2]; //2 vars selected
record[1] = id_list[i]; //for the pre var
record[0] = id_list[i + 1]; //for the s/o var
result_list.push_back(record);
}
}
else if (total_num == 0) //only ?p
{
//just use so2p
int sid = (this->kvstore)->getIDByEntity(triple.subject);
int oid = (this->kvstore)->getIDByEntity(triple.object);
if (oid == -1)
{
oid = (this->kvstore)->getIDByLiteral(triple.object);
}
this->kvstore->getpreIDlistBysubIDobjID(sid, oid, id_list, id_list_len);
//copy to result list
for (int i = 0; i < id_list_len; ++i)
{
int* record = new int[1];
record[0] = id_list[i];
result_list.push_back(record);
}
}
delete[] id_list;
continue;
this->method = 4;
}
if (pre_varNum == 0 && (*iter)->getTripleNum() == 1) //only one triple and no predicates
if (this->method < 0 && pre_varNum == 0 && (*iter)->getTripleNum() == 1) //only one triple and no predicates
{
//only one variable and one triple: ?s pre obj or sub pre ?o
if (total_num == 1)
@ -150,16 +92,22 @@ Strategy::handle(SPARQLquery& _query, ResultFilter* _result_filter)
this->method = 3;
}
}
//cerr << "this BasicQuery use query strategy 2" << endl;
//cerr<<"Final result size: "<<(*iter)->getResultList().size()<<endl;
//cout << "this BasicQuery use query strategy 2" << endl;
//cout<<"Final result size: "<<(*iter)->getResultList().size()<<endl;
//continue;
}
if(this->method< 0)
{
this->method = 0;
}
//QueryHandler dispatch;
//dispatch[0] = handler0;
switch (this->method)
{
//BETTER: use function pointer array in C++ class
case 0:
//default:filter by vstree and then verified by join
this->handler0(*iter, result_list, _result_filter);
break;
case 1:
@ -171,21 +119,23 @@ Strategy::handle(SPARQLquery& _query, ResultFilter* _result_filter)
case 3:
this->handler3(*iter, result_list);
break;
case 4:
this->handler4(*iter, result_list);
break;
default:
cerr << "not support this method" << endl;
cout << "not support this method" << endl;
}
cerr << "Final result size: " << (*iter)->getResultList().size() << endl;
//BETTER: use function pointer array in C++ class
cout << "BasicQuery -- Final result size: " << (*iter)->getResultList().size() << endl;
}
#else
cerr << "this BasicQuery use original query strategy" << endl;
cout << "this BasicQuery use original query strategy" << endl;
long tv_handle = Util::get_cur_time();
(this->vstree)->retrieve(_query);
long tv_retrieve = Util::get_cur_time();
cout << "after Retrieve, used " << (tv_retrieve - tv_handle) << "ms." << endl;
this->join = new Join(kvstore, pre2num);
this->join = new Join(kvstore, pre2num, this->limitID_predicate, this->limitID_literal);
this->join->join_sparql(_query);
delete this->join;
@ -200,7 +150,7 @@ void
Strategy::handler0(BasicQuery* _bq, vector<int*>& _result_list, ResultFilter* _result_filter)
{
//long before_filter = Util::get_cur_time();
cerr << "this BasicQuery use query strategy 0" << endl;
cout << "this BasicQuery use query strategy 0" << endl;
//BETTER:not all vars in join filtered by vstree
//(A)-B-c: B should by vstree, then by c, but A should be generated in join(first set A as not)
@ -226,6 +176,7 @@ Strategy::handler0(BasicQuery* _bq, vector<int*>& _result_list, ResultFilter* _r
this->vstree->retrieveEntity(entityBitSet, idListPtr);
if (!flag)
{
//cout<<"set ready: "<<i<<endl;
_bq->setReady(i);
}
//the basic query should end if one non-literal var has no candidates
@ -235,8 +186,6 @@ Strategy::handler0(BasicQuery* _bq, vector<int*>& _result_list, ResultFilter* _r
}
}
//if(_bq->isReady(0))
//cout<<"error: var 0 is ready?"<<endl;
//TODO:end directly if one is empty!
long tv_retrieve = Util::get_cur_time();
@ -246,7 +195,7 @@ Strategy::handler0(BasicQuery* _bq, vector<int*>& _result_list, ResultFilter* _r
if (_result_filter != NULL)
_result_filter->candFilterWithResultHashTable(*_bq);
Join *join = new Join(kvstore, pre2num);
Join *join = new Join(kvstore, pre2num, this->limitID_predicate, this->limitID_literal);
join->join_basic(_bq);
delete join;
@ -258,7 +207,7 @@ void
Strategy::handler1(BasicQuery* _bq, vector<int*>& _result_list)
{
long before_filter = Util::get_cur_time();
cerr << "this BasicQuery use query strategy 1" << endl;
cout << "this BasicQuery use query strategy 1" << endl;
//int neighbor_id = (*_bq->getEdgeNeighborID(0, 0); //constant, -1
char edge_type = _bq->getEdgeType(0, 0);
int triple_id = _bq->getEdgeID(0, 0);
@ -268,7 +217,7 @@ Strategy::handler1(BasicQuery* _bq, vector<int*>& _result_list)
int id_list_len = 0;
if (edge_type == Util::EDGE_OUT)
{
//cerr<<"edge out!!!"<<endl;
//cout<<"edge out!!!"<<endl;
int nid = (this->kvstore)->getIDByEntity(triple.object);
if (nid == -1)
{
@ -278,32 +227,32 @@ Strategy::handler1(BasicQuery* _bq, vector<int*>& _result_list)
}
else
{
//cerr<<"edge in!!!"<<endl;
//cout<<"edge in!!!"<<endl;
this->kvstore->getobjIDlistBysubIDpreID(this->kvstore->getIDByEntity(triple.subject), pre_id, id_list, id_list_len);
}
long after_filter = Util::get_cur_time();
cerr << "after filter, used " << (after_filter - before_filter) << "ms" << endl;
cout << "after filter, used " << (after_filter - before_filter) << "ms" << endl;
_result_list.clear();
//cerr<<"now to copy result to list"<<endl;
//cout<<"now to copy result to list"<<endl;
for (int i = 0; i < id_list_len; ++i)
{
int* record = new int[1]; //only this var is selected
record[0] = id_list[i];
//cerr<<this->kvstore->getEntityByID(record[0])<<endl;
//cout<<this->kvstore->getEntityByID(record[0])<<endl;
_result_list.push_back(record);
}
long after_copy = Util::get_cur_time();
cerr << "after copy to result list: used " << (after_copy - after_filter) << " ms" << endl;
cout << "after copy to result list: used " << (after_copy - after_filter) << " ms" << endl;
delete[] id_list;
cerr << "Final result size: " << _result_list.size() << endl;
cout << "Final result size: " << _result_list.size() << endl;
}
void
Strategy::handler2(BasicQuery* _bq, vector<int*>& _result_list)
{
long before_filter = Util::get_cur_time();
cerr << "this BasicQuery use query strategy 2" << endl;
cout << "this BasicQuery use query strategy 2" << endl;
int triple_id = _bq->getEdgeID(0, 0);
Triple triple = _bq->getTriple(triple_id);
int pre_id = _bq->getEdgePreID(0, 0);
@ -326,10 +275,10 @@ Strategy::handler2(BasicQuery* _bq, vector<int*>& _result_list)
}
else
{
cerr << "ERROR in Database::handle(): no selected var!" << endl;
cout << "ERROR in Database::handle(): no selected var!" << endl;
}
long after_filter = Util::get_cur_time();
cerr << "after filter, used " << (after_filter - before_filter) << "ms" << endl;
cout << "after filter, used " << (after_filter - before_filter) << "ms" << endl;
_result_list.clear();
for (int i = 0; i < id_list_len; ++i)
{
@ -338,16 +287,16 @@ Strategy::handler2(BasicQuery* _bq, vector<int*>& _result_list)
_result_list.push_back(record);
}
long after_copy = Util::get_cur_time();
cerr << "after copy to result list: used " << (after_copy - after_filter) << " ms" << endl;
cout << "after copy to result list: used " << (after_copy - after_filter) << " ms" << endl;
delete[] id_list;
cerr << "Final result size: " << _result_list.size() << endl;
cout << "Final result size: " << _result_list.size() << endl;
}
void
Strategy::handler3(BasicQuery* _bq, vector<int*>& _result_list)
{
long before_filter = Util::get_cur_time();
cerr << "this BasicQuery use query strategy 3" << endl;
cout << "this BasicQuery use query strategy 3" << endl;
int triple_id = _bq->getEdgeID(0, 0);
Triple triple = _bq->getTriple(triple_id);
int pre_id = _bq->getEdgePreID(0, 0);
@ -356,8 +305,8 @@ Strategy::handler3(BasicQuery* _bq, vector<int*>& _result_list)
_result_list.clear();
this->kvstore->getsubIDobjIDlistBypreID(pre_id, id_list, id_list_len);
int var1_id = _bq->getIDByVarName(triple.subject);
int var2_id = _bq->getIDByVarName(triple.object);
int var1_id = _bq->getSelectedVarPosition(triple.subject);
int var2_id = _bq->getSelectedVarPosition(triple.object);
if(var1_id < 0 || var2_id < 0)
{
@ -366,7 +315,7 @@ Strategy::handler3(BasicQuery* _bq, vector<int*>& _result_list)
}
long after_filter = Util::get_cur_time();
cerr << "after filter, used " << (after_filter - before_filter) << "ms" << endl;
cout << "after filter, used " << (after_filter - before_filter) << "ms" << endl;
for (int i = 0; i < id_list_len; i += 2)
{
@ -377,7 +326,134 @@ Strategy::handler3(BasicQuery* _bq, vector<int*>& _result_list)
}
long after_copy = Util::get_cur_time();
cerr << "after copy to result list: used " << (after_copy - after_filter) << " ms" << endl;
cout << "after copy to result list: used " << (after_copy - after_filter) << " ms" << endl;
delete[] id_list;
cerr << "Final result size: " << _result_list.size() << endl;
cout << "Final result size: " << _result_list.size() << endl;
}
void
Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
{
cout<<"Special Case: consider pre var in this triple"<<endl;
int varNum = _bq->getVarNum();
//all variables(not including pre vars)
int total_num = _bq->getTotalVarNum();
int pre_varNum = _bq->getPreVarNum();
int selected_pre_var_num = _bq->getSelectedPreVarNum();
int selected_var_num = _bq->getSelectVarNum();
Triple triple = _bq->getTriple(0);
int pvpos = _bq->getSelectedPreVarPosition(triple.predicate);
int* id_list = NULL;
int id_list_len = 0;
_result_list.clear();
//cout<<"total num: "<<total_num <<endl;
if (total_num == 2)
{
cout<<"Special Case 1"<<endl;
int svpos = _bq->getSelectedVarPosition(triple.subject);
int ovpos = _bq->getSelectedVarPosition(triple.object);
cout<<"subject: "<<triple.subject<<" "<<svpos<<endl;
cout<<"object: "<<triple.object<<" "<<ovpos<<endl;
cout<<"predicate: "<<triple.predicate<<" "<<pvpos<<endl;
//very special case, to find all triples, select ?s (?p) ?o where { ?s ?p ?o . }
//filter and join is too costly, should enum all predicates and use p2so
for(int i = 0; i < this->limitID_predicate; ++i)
{
int pid = i;
this->kvstore->getsubIDobjIDlistBypreID(pid, id_list, id_list_len);
int rsize = selected_var_num;
if(selected_pre_var_num == 1)
{
rsize++;
}
//always place s/o before p in result list
for (int j = 0; j < id_list_len; j += 2)
{
int* record = new int[rsize];
//check the s/o var if selected, need to ensure the placement order
if(ovpos >= 0)
{
record[ovpos] = id_list[j+1];
}
if(svpos >= 0)
{
record[svpos] = id_list[j];
}
if(pvpos >= 0)
{
record[pvpos] = pid; //for the pre var
}
_result_list.push_back(record);
}
delete[] id_list;
}
id_list = NULL;
}
else if (total_num == 1)
{
cout<<"Special Case 2"<<endl;
int vpos = -1;
if(triple.subject[0] != '?') //constant
{
int sid = (this->kvstore)->getIDByEntity(triple.subject);
this->kvstore->getpreIDobjIDlistBysubID(sid, id_list, id_list_len);
vpos = _bq->getSelectedVarPosition(triple.object);
}
else if(triple.object[0] != '?') //constant
{
int oid = (this->kvstore)->getIDByEntity(triple.object);
if (oid == -1)
{
oid = (this->kvstore)->getIDByLiteral(triple.object);
}
this->kvstore->getpreIDsubIDlistByobjID(oid, id_list, id_list_len);
vpos = _bq->getSelectedVarPosition(triple.subject);
}
int rsize = varNum;
if(selected_pre_var_num == 1)
{
rsize++;
}
//always place s/o before p in result list
for (int i = 0; i < id_list_len; i += 2)
{
int* record = new int[rsize];
if(vpos >= 0)
{
record[vpos] = id_list[i + 1]; //for the s/o var
}
if(pvpos >= 0)
{
record[pvpos] = id_list[i]; //for the pre var
}
_result_list.push_back(record);
}
}
else if (total_num == 0) //only ?p and it must be selected
{
cout<<"Special Case 3"<<endl;
//just use so2p
int sid = (this->kvstore)->getIDByEntity(triple.subject);
int oid = (this->kvstore)->getIDByEntity(triple.object);
if (oid == -1)
{
oid = (this->kvstore)->getIDByLiteral(triple.object);
}
this->kvstore->getpreIDlistBysubIDobjID(sid, oid, id_list, id_list_len);
//copy to result list
for (int i = 0; i < id_list_len; ++i)
{
int* record = new int[1];
record[0] = id_list[i];
_result_list.push_back(record);
}
}
delete[] id_list;
}

6
Database/Strategy.h
View File

@ -23,7 +23,7 @@ class Strategy
{
public:
Strategy();
Strategy(KVstore*, VSTree*, TNUM*);
Strategy(KVstore*, VSTree*, TNUM*, int, int);
~Strategy();
//select efficient strategy to do the sparql query
bool handle(SPARQLquery&, ResultFilter* _result_filter = NULL);
@ -33,10 +33,14 @@ private:
KVstore* kvstore;
VSTree* vstree;
TNUM* pre2num;
int limitID_predicate;
int limitID_literal;
void handler0(BasicQuery*, vector<int*>&, ResultFilter* _result_filter = NULL);
void handler1(BasicQuery*, vector<int*>&);
void handler2(BasicQuery*, vector<int*>&);
void handler3(BasicQuery*, vector<int*>&);
void handler4(BasicQuery*, vector<int*>&);
//QueryHandler *dispatch;
//void prepare_handler();
};

3
KVstore/KVstore.cpp
View File

@ -1108,6 +1108,7 @@ string KVstore::getLiteralByID(int _id) const {
int _len = 0;
bool _get = this->getValueByKey(this->id2literal, _id, _tmp, _len);
if (!_get) {
//NOTICE:here assumes that all literals cannot be empty: ""
return "";
}
string _ret = string(_tmp);
@ -1971,4 +1972,4 @@ unsigned short KVstore::buffer_oID2values_build = 32;
unsigned short KVstore::buffer_pID2values_build = 16;
unsigned short KVstore::buffer_sID2values_query = 16;
unsigned short KVstore::buffer_oID2values_query = 16;
unsigned short KVstore::buffer_pID2values_query = 8;
unsigned short KVstore::buffer_pID2values_query = 8;

7
Main/gbuild.cpp
View File

@ -20,6 +20,13 @@ main(int argc, char * argv[])
#ifdef DEBUG
Util util;
#endif
if(argc == 1) //./gbuild
{
//output help info here
cout << "the usage of gbuild: " << endl;
cout << "bin/gbuild your_database_name.db your_dataset_path" << endl;
return 0;
}
//system("clock");
cout << "gbuild..." << endl;
{

87
Main/gconsole.cpp
View File

@ -114,7 +114,6 @@ FILE* output = stdout;
//current using database in local
Database *current_database = NULL;
//TODO:redirect 2>&1 or adjust the fprintf->stderr to file pointer
int
main(int argc, char **argv)
{
@ -128,34 +127,35 @@ main(int argc, char **argv)
char *line, *s;
progname = argv[0];
long cur_path_len;
char* cur_work_dir;
if ((cur_path_len = pathconf(".", _PC_PATH_MAX)) == -1) {
cerr << "Couldn't get current working path length" << endl;
return 1;
}
if ((cur_work_dir = (char*)malloc(cur_path_len + 1 + strlen(progname))) == NULL) {
cerr << "Couldn't allocate memory for the pathname" << endl;
return 1;
}
if (getcwd(cur_work_dir, cur_path_len) == NULL) {
cerr << "Couldn't get current working directory!" << endl;
return 1;
}
strcat(cur_work_dir, "/");
strcat(cur_work_dir, progname);
char* dir_name = dirname(cur_work_dir);
free(cur_work_dir);
if (dir_name == NULL) {
cerr << "Couldn't get current directory name!" << endl;
return 1;
}
string root(dir_name);
root += "/..";
if (chdir(root.c_str()) == -1) {
cerr << "Couldn't change current directory!" << endl;
return 1;
}
//NOTICE+DEBUG: the code segment below is used to change directory if not in gStore root, but here exists some bugs
//long cur_path_len;
//char* cur_work_dir;
//if ((cur_path_len = pathconf(".", _PC_PATH_MAX)) == -1) {
//cerr << "Couldn't get current working path length" << endl;
//return 1;
//}
//if ((cur_work_dir = (char*)malloc(cur_path_len + 1 + strlen(progname))) == NULL) {
//cerr << "Couldn't allocate memory for the pathname" << endl;
//return 1;
//}
//if (getcwd(cur_work_dir, cur_path_len) == NULL) {
//cerr << "Couldn't get current working directory!" << endl;
//return 1;
//}
//strcat(cur_work_dir, "/");
//strcat(cur_work_dir, progname);
//char* dir_name = dirname(cur_work_dir);
//free(cur_work_dir);
//if (dir_name == NULL) {
//cerr << "Couldn't get current directory name!" << endl;
//return 1;
//}
//string root(dir_name);
//root += "/..";
//if (chdir(root.c_str()) == -1) {
//cerr << "Couldn't change current directory!" << endl;
//return 1;
//}
system("clear");
@ -1171,4 +1171,31 @@ int printport_handler(const vector<string>& args) {
return -1;
}
return system("bin/gserver -P");
}
}

79
Main/gquery.cpp
View File

@ -50,7 +50,7 @@ main(int argc, char * argv[])
cout << "gquery..." << endl;
if (argc < 2)
{
cerr << "error: lack of DB_store to be queried" << endl;
cout << "error: lack of DB_store to be queried" << endl;
return 0;
}
{
@ -100,11 +100,48 @@ main(int argc, char * argv[])
}
printf("query is:\n%s\n\n", query.c_str());
ResultSet _rs;
_db.query(query, _rs, stdout);
FILE* ofp = stdout;
if (argc >= 4)
{
Util::save_to_file(argv[3], _rs.to_str());
ofp = fopen(argv[3], "w");
}
string msg;
int ret = _db.query(query, _rs, ofp);
if(argc >= 4)
{
fclose(ofp);
ofp = NULL;
}
//cout<<"gquery ret: "<<ret<<endl;
if (ret <= -100) //select query
{
if(ret == -100)
{
msg = _rs.to_str();
}
else //query error
{
msg = "query failed.";
}
}
else //update query
{
if(ret >= 0)
{
msg = "update num: " + Util::int2string(ret);
}
else //update error
{
msg = "update failed.";
}
}
if(ret != -100)
{
cout << msg <<endl;
}
return 0;
}
@ -220,7 +257,39 @@ main(int argc, char * argv[])
printf("query is:\n");
printf("%s\n\n", query.c_str());
ResultSet _rs;
_db.query(query, _rs, fp);
int ret = _db.query(query, _rs, fp);
//int ret = _db.query(query, _rs, NULL);
string msg;
//cout<<"gquery ret: "<<ret<<endl;
if (ret <= -100) //select query
{
if(ret == -100)
{
msg = "";
}
else //query error
{
msg = "query failed.";
}
}
else //update query
{
if(ret >= 0)
{
msg = "update num: " + Util::int2string(ret);
}
else //update error
{
msg = "update failed.";
}
}
if(ret != -100)
{
cout << msg << endl;
}
//test...
//string answer_file = query_file+".out";
//Util::save_to_file(answer_file.c_str(), _rs.to_str());
@ -255,4 +324,4 @@ main(int argc, char * argv[])
#endif // READLINE_ON
return 0;
}
}

32
Main/gserver.cpp
View File

@ -35,7 +35,7 @@ int main(int argc, char* argv[])
}
if (argc > 3 || (argc == 3 && mode != "-p" && mode != "--port")) {
cerr << "Invalid arguments! Input \"bin/gserver -h\" for help." << endl;
cout << "Invalid arguments! Input \"bin/gserver -h\" for help." << endl;
return -1;
}
@ -61,7 +61,7 @@ int main(int argc, char* argv[])
unsigned short port = Socket::DEFAULT_CONNECT_PORT;
if (argc == 3) {
if (!Util::isValidPort(string(argv[2]))) {
cerr << "Invalid port: " << argv[2] << endl;
cout << "Invalid port: " << argv[2] << endl;
return -1;
}
else {
@ -72,7 +72,7 @@ int main(int argc, char* argv[])
if (!isOnlyProcess(argv[0])) {
ofstream out(GSERVER_PORT_SWAP, ios::out);
if (!out) {
cerr << "Failed to change port!" << endl;
cout << "Failed to change port!" << endl;
return -1;
}
out << port;
@ -82,7 +82,7 @@ int main(int argc, char* argv[])
}
ofstream out(GSERVER_PORT_FILE, ios::out);
if (!out) {
cerr << "Failed to change port!" << endl;
cout << "Failed to change port!" << endl;
return -1;
}
out << port;
@ -93,7 +93,7 @@ int main(int argc, char* argv[])
if (mode == "-s" || mode == "--start") {
if (!isOnlyProcess(argv[0])) {
cerr << "gServer already running!" << endl;
cout << "gServer already running!" << endl;
return -1;
}
if (startServer()) {
@ -106,7 +106,7 @@ int main(int argc, char* argv[])
if (mode == "-t" || mode == "--stop") {
if (isOnlyProcess(argv[0])) {
cerr << "gServer not running!" << endl;
cout << "gServer not running!" << endl;
return -1;
}
if (stopServer()) {
@ -119,7 +119,7 @@ int main(int argc, char* argv[])
if (mode == "-r" || mode == "--restart") {
if (isOnlyProcess(argv[0])) {
cerr << "gServer not running!" << endl;
cout << "gServer not running!" << endl;
return -1;
}
if (!stopServer()) {
@ -153,14 +153,14 @@ int main(int argc, char* argv[])
if (mode == "-k" || mode == "--kill") {
if (isOnlyProcess(argv[0])) {
cerr << "No process to kill!" << endl;
cout << "No process to kill!" << endl;
return -1;
}
execl("/usr/bin/killall", "killall", Util::getExactPath(argv[0]).c_str(), NULL);
return 0;
}
cerr << "Invalid arguments! Input \"bin/gserver -h\" for help." << endl;
cout << "Invalid arguments! Input \"bin/gserver -h\" for help." << endl;
return -1;
}
@ -174,7 +174,7 @@ void checkSwap() {
}
ifstream in(GSERVER_PORT_SWAP, ios::in);
if (!in) {
cerr << "Failed in checkSwap(), port may not be changed." << endl;
cout << "Failed in checkSwap(), port may not be changed." << endl;
return;
}
unsigned short port;
@ -182,7 +182,7 @@ void checkSwap() {
in.close();
ofstream out(GSERVER_PORT_FILE, ios::out);
if (!out) {
cerr << "Failed in checkSwap(), port may not be changed." << endl;
cout << "Failed in checkSwap(), port may not be changed." << endl;
return;
}
out << port;
@ -219,7 +219,7 @@ bool startServer() {
freopen(GSERVER_LOG, "a", stderr);
Server server(port);
if (!server.createConnection()) {
cerr << Util::getTimeString() << "Failed to create connection at port " << port << '.' << endl;
cout << Util::getTimeString() << "Failed to create connection at port " << port << '.' << endl;
return false;
}
cout << Util::getTimeString() << "Server started at port " << port << '.' << endl;
@ -234,7 +234,7 @@ bool startServer() {
}
// fork failure
else {
cerr << "Failed to start server at port " << port << '.' << endl;
cout << "Failed to start server at port " << port << '.' << endl;
return false;
}
}
@ -248,17 +248,17 @@ bool stopServer() {
}
Socket socket;
if (!socket.create() || !socket.connect("127.0.0.1", port) || !socket.send("stop")) {
cerr << "Failed to stop server at port " << port << '.' << endl;
cout << "Failed to stop server at port " << port << '.' << endl;
return false;
}
string recv_msg;
socket.recv(recv_msg);
socket.close();
if (recv_msg != "server stopped.") {
cerr << "Failed to stop server at port " << port << '.' << endl;
cout << "Failed to stop server at port " << port << '.' << endl;
return false;
}
cout << "Server stopped at port " << port << '.' << endl;
checkSwap();
return true;
}
}

148
NOTES.md
View File

@ -1,38 +1,95 @@
# NOTICE
一般在实践应用中不是单机，而是CS模式，因为API也是基于CS模式的。
那么在CS模式中，可以禁用server端对query结果的输出，提升效率
方式是控制Util/Util.h中OUTPUT_QUERY_RESULT宏的开启
在使用gserver时，不能在数据库没有unload时再用gbuild或其他命令修改数据库，仅限于C/S模式
# TODO
gclient 不能正常使用
gserver for multiple users, not load db for each connection, but get the db pointer
assign the pointer to the user who require this db, and ensure that each db is just kept once in memory
成组插删还没完全支持和测试，需要修改KVstore.cpp
支持返回?p结果的查询
辛雨非的插入时vstree bug和彭鹏的build时vstree bug
vstree部分在build和insert时候的错误
统计74上的8亿测试结果
控制台命令如果使用方式不当，比如直接gbuild，应该输出提示信息示意该怎么操作
数据库的多版本备份，动态删除
保持连接而不是每次都用socket
模仿海量图大作业，基于gStore开发一个社交应用，要求可以批量导入且实时查询
gstore后续需要开发的地方：
数据库连接池 保持连接而不是每次都用socket
事务操作
安全备份 数据库的多版本备份，动态删除
多领域多库解决方案。
分页查询
任务分配：
---
JSON格式传输(陈佳棋)
陈佳棋的任务：s和p对应同一个实体，应该先重命名，再过滤。还有一种情况是两者只是名字相同，实则并无关系
---
陈佳棋找人负责：
模仿海量图大作业，基于gStore开发一个社交应用，要求可以批量导入且实时查询
查询级别的缓存（测试时可将查询复制后再随机打乱）
多查询优化
---
王力博：
gserver for multiple users, not load db for each connection, but get the same db pointer
assign the pointer to the user who require this db, and ensure that each db is just kept once in memory
有什么办法去检测一个db是否存在呢？(首先要支持导入多个数据库)
如果不存在 就新建一个 再进行查询 如果存在 就直接进行查询
gserver-gclient if gclient quit(without unload), then restart, there maybe exist too many gserver process(so 3305 is occupied)
or the reason maybe gserver still dealing with the previous job, then a new connection and a new job comes
如何避免整个server崩溃或卡死，无论单个查询/建立等操作遇到任何问题
---
胡琳：
彭鹏师兄的数据集bug
优化谓词查询，谓词少而entity/literal多，所以先过滤得到谓词的解是一种可以考虑的策略
以谓词为节点，以s/o为信息，来过滤得到谓词的结果
---
陈语嫣，李荆
---
张雨的任务：单起点单终点的正则表达式路径问题，如果是多起点多终点？
下学期的任务：提取相关联的几个方向写论文
将IRC聊天放到gstore文档上，freenode #gStore
WARN：B+树删除时，向旁边兄弟借或者合并，要注意兄弟的定义，应该是同一父节点才对！
考虑使用 sigmod2016 那篇图同构的论文方法，实现一套join
但那个是基于路径的，BFS和索引连接的思想值得借用，可作为另外一套join方法
@hulin
---
实现其他的join思路，比如基于过滤效果
pthread写多线程
如何在preFilter和join的开销之间做平衡
preFilter中的限制条件是否过于严格
寻找查询图的特征，分类做查询计划：
先对于每个查询，确定各部分的开销比例
fix the full_test:how to sum the db size right?
for virtuoso, better to reset each time and the given configure file(need to reserver the temp logs)
load过程先导入满足内存的内容，或者先来几轮搜索但不输出结果，避免开头的查询要读磁盘。vstree直接全导入内存？
先完成合并测试，再测lubm500M和bsbm500M -- 90 server
两表拼接时多线程分块join
jemalloc??
vstree并行分块
@lijing
考虑出入度数，编码为1的个数？应该不用，在编码的邻居信息中能够得到体现。
第二步编码应该更长，点和边对应着放在一起。按出入边分区，一步点，二步边分区和二步点。
对一个节点保留其最长链信息没啥用，因为数据图基本是连通的，最长链就是图的最长链。
多步编码不应分开，而应在编码逐一扩展，第二步可以依旧保留详细信息，最好用更长编码，因为信息更多。
可能有相同的谓词对应多个邻居，同样的谓词只要保留一个即可，不同邻居可以重合。
第三步可以只记谓词，第四步可以只记边的出入向。
各版本对比的表格中应加一列几何平均数，现实中大多数查询是简单查询，最好还有一个平均数，对应着把数据做归一化后求和
dbpedia q6现在应该统计并对比结果了，包含在测试结果中
最好用json格式来输出结果，或者在JSON和字符串之间相互转换
另外在改善kvstore后，build过程明显加快了很多，现在vstree的建立时间成了瓶颈
在改善kvstore后，build过程明显加快了很多，现在vstree的建立时间成了瓶颈
preFilter中不仅要看谓词，也要看表大小以及度数，有些节点最好过滤！！！
在0.4.0版本后统一使用git管理，为各位开发者维护一个分支
更新github上的测试报告和help文档
5亿+bug，最好单机能支持到10亿
---
备份原来的代码后，合并王的代码并测试，再在原来代码上加join缓存测试
以后统一用git进行版本管理，以当前版本为基础
bookug, hulin, wlb, cjq, pp, gpu, wuda, chenyuyan各一个分支
张雨的任务：单起点单终点的正则表达式路径问题，如果是多起点多终点？
下学期的任务：提取相关联的几个方向写论文
将IRC聊天放到gstore文档上，freenode #gStore
允许同时使用多个数据库，查询时指明数据库，这样的话可以支持from，但各个数据库是相互独立的；
添加scala API;
git管理开发；
从下往上建立B+树和vstree树
加快更新操作，如insert和delete等，是否考虑增加修改的源操作
@ -49,50 +106,11 @@ bookug, hulin, wlb, cjq, pp, gpu, wuda, chenyuyan各一个分支
但机器学习对新问题没有一个基本的界，不像数学化的评估函数，对任何情况都能保证一个上界
(目前数据库里面很少有人利用机器学习来估价)
WARN：B+树删除时，向旁边兄弟借或者合并，要注意兄弟的定义，应该是同一父节点才对！
输出还是组织成JSON格式比较好，用JSON的C++库进行压缩解压，客户端也无需对输出格式有太多了解，减轻后者负担
URI是唯一的，RDF图中不存在相同标签的节点，无法合并相似节点，也没法利用自同构的预处理加快查询
DBpedia最新的数据集，原来的相对太小了
count函数的优化：深搜计数即可，不必生成中间表
B+树每个节点内部添加索引，对keys分段？
考虑使用 sigmod2016 那篇图同构的论文方法，实现一套join
但那个是基于路径的，BFS和索引连接的思想值得借用，可作为另外一套join方法
@hulin
---
实现其他的join思路，比如基于过滤效果
pthread写多线程
可以尝试开-O6优化，如果结果保证正确，应该速度可以提升很多
@lijing
验证-O6的正确性和效率
如何在preFilter和join的开销之间做平衡
preFilter中的限制条件是否过于严格
寻找查询图的特征，分类做查询计划：
先对于每个查询，确定各部分的开销比例
fix the full_test:how to sum the db size right?
for virtuoso, better to reset each time and the given configure file(need to reserver the temp logs)
load过程先导入满足内存的内容，或者先来几轮搜索但不输出结果，避免开头的查询要读磁盘。vstree直接全导入内存？
先完成合并测试，再测lubm500M和bsbm500M -- 90 server
两表拼接时多线程分块join
jemalloc??
查询级别的缓存（测试时可将查询复制后再随机打乱）
vstree并行分块
@lijing
考虑出入度数，编码为1的个数？应该不用，在编码的邻居信息中能够得到体现。
第二步编码应该更长，点和边对应着放在一起。按出入边分区，一步点，二步边分区和二步点。
对一个节点保留其最长链信息没啥用，因为数据图基本是连通的，最长链就是图的最长链。
多步编码不应分开，而应在编码逐一扩展，第二步可以依旧保留详细信息，最好用更长编码，因为信息更多。
可能有相同的谓词对应多个邻居，同样的谓词只要保留一个即可，不同邻居可以重合。
第三步可以只记谓词，第四步可以只记边的出入向。
是否可以考虑把literal也作为vstree中的节点，加快过滤？
join过程中生成literal可能开销大，且可以考虑用周围边对literal进行过滤。。。
@ -365,7 +383,7 @@ http://www.oschina.net/question/188977_58777
- 新算法在冷启动时时间不理想，即便只是0轮join开销也很大，对比时采用热启动
- 不能支持非连通图查询(应该分割为多个连通图)
- 如果select中某变量未出现在查询图中，应该对该变量返回空值还是直接认为该查询非法
- - -

242
Query/BasicQuery.cpp
View File

@ -141,20 +141,33 @@ BasicQuery::getVarDegree(int _var)
//get the index of edge between two var ids
//-1 if no edge, the index is based on _id0
int
vector<int>
BasicQuery::getEdgeIndex(int _id0, int _id)
{
//BETTER:this a brute-force method, using mapping if needed
//(though the query graph is small generally)
//is there any way to pre-compute all EdgeIndex relationship
vector<int> result;
for(int i = 0; i < this->var_degree[_id0]; ++i)
{
if(this->edge_nei_id[_id0][i] == _id)
{
return i;
//NOTICE:maybe exist parallel edges i.e.
//?s ?p1 ?o. ?s ?p2 ?o.
//?s p1 ?o. ?s p2 ?o.
//return i;
//if(this->getEdgePreID(_id0, i) < 0)
//{
//continue;
//}
result.push_back(i);
}
}
return -1;
//return -1;
return result;
}
//get the candidate list of _var in the query graph
@ -274,7 +287,10 @@ BasicQuery::isFreeLiteralVariable(int _var)
int neighbor_num = this->var_degree[_var];
for(int i = 0; i < neighbor_num; i ++)
{
if(this->edge_nei_id[_var][i] == -1)
int triple_id = this->getEdgeID(_var, i);
Triple triple = this->getTriple(triple_id);
//NOTICE: the neighbor can also be a var whose id is -1
if(triple.subject[0] != '?')
{
return false;
}
@ -379,13 +395,8 @@ bool
BasicQuery::encodeBasicQuery(KVstore* _p_kvstore, const vector<string>& _query_var)
{
//WARN:?p is ok to exist in both s/o or p position
//return only the entity ID, not the predicate ID
//TODO:return only the entity ID, not the predicate ID?
//so the _query_var mix all the variables, not considering the order
//
//TODO:the third parameter should be selected predicate variables
//(ordered, and merged with selected s/o in upper level)
//we append the candidates for selected pre_var to original select_var_num columns
//TODO:add pre var, assign name and select=true (not disturb the order between pres)
cout << "IN buildBasicSignature" << endl;
//this->initial();
//cout << "after init" << endl;
@ -395,20 +406,58 @@ BasicQuery::encodeBasicQuery(KVstore* _p_kvstore, const vector<string>& _query_v
// map id 2 var_name : this->var_name[]
// map var_name 2 id : this->var_str2id
this->select_var_num = _query_var.size();
//find all predicate vars
for(unsigned i = 0; i < this->triple_vt.size(); i ++)
{
string& pre = this->triple_vt[i].predicate;
if(pre[0] != '?')
{
continue;
}
string& sub = this->triple_vt[i].subject;
string& obj = this->triple_vt[i].object;
int pid = this->getPreVarID(pre);
if(pid == -1)
{
this->pre_var.push_back(PreVar(pre));
pid = this->pre_var.size() - 1;
}
//map<int, PreVar>::iterator it = this->pre_var.find(pid);
//it->second.triples.push_back(i);
this->pre_var[pid].triples.push_back(i);
}
//NOTICE: we append the candidates for selected pre_var to original select_var_num columns
this->select_var_num = this->selected_pre_var_num = 0;
cout<<"now to check the query var list order:"<<endl;
for(unsigned i = 0; i < _query_var.size(); ++i)
{
//NOTICE:not place pre var in join
string var = _query_var[i];
cout<<i<<" "<<var<<endl;
int pid = this->getPreVarID(var);
if(pid == -1) // not pre var
{
this->selected_var_set.insert(var);
//assign id for this var
int id = this->select_var_num;
this->selected_var_position[id] = i;
this->var_str2id[var] = id;
this->var_name[id] = var;
this->select_var_num++;
continue;
}
this->selected_var_position[-1-pid] = i;
this->pre_var[pid].selected = true;
this->selected_pre_var_num++;
}
#ifdef DEBUG
stringstream _ss;
_ss << "select_var_num=" << this->select_var_num << endl;
Util::logging(_ss.str());
#endif
for(int i = 0; i < (this->select_var_num); ++i)
{
//NOTICE:not place pre var in join
string _var = _query_var[i];
this->var_str2id[_var] = i;
this->var_name[i] = _var;
}
cout << "select variables: ";
for(unsigned i = 0; i < this->var_str2id.size(); ++i)
@ -416,6 +465,7 @@ BasicQuery::encodeBasicQuery(KVstore* _p_kvstore, const vector<string>& _query_v
cout << "[" << this->var_name[i] << ", " << i << " " << this->var_str2id[this->var_name[i]] << "]\t";
}
cout << endl;
//BETTER:ouput the selected pre vars
this->total_var_num = this->select_var_num;
if(this->encode_method == BasicQuery::SELECT_VAR)
@ -444,20 +494,10 @@ BasicQuery::encodeBasicQuery(KVstore* _p_kvstore, const vector<string>& _query_v
string& sub = this->triple_vt[i].subject;
string& pre = this->triple_vt[i].predicate;
string& obj = this->triple_vt[i].object;
int pre_id = -1; //not found
if(pre[0] == '?') //pre var
{
int pid = this->getPreVarID(pre);
if(pid == -1)
{
//pid = this->select_var_num + this->pre_var.size();
//this->pre_var[pid] = PreVar(pre);
this->pre_var.push_back(PreVar(pre));
pid = this->pre_var.size() - 1;
}
//map<int, PreVar>::iterator it = this->pre_var.find(pid);
//it->second.triples.push_back(i);
this->pre_var[pid].triples.push_back(i);
pre_id = -2; //mark that this is a pre var
}
else
@ -469,18 +509,15 @@ BasicQuery::encodeBasicQuery(KVstore* _p_kvstore, const vector<string>& _query_v
_ss << "pre2id: " << pre << "=>" << pre_id << endl;
Util::logging(_ss.str());
}
if(pre_id == -1)
{
//TODO:end this query, otherwise maybe error later
cerr << "invalid query because the pre is not found: " << pre << endl;
//BETTER:this is too robust, not only one query, try return false
//exit(1);
return false;
}
}
if(pre_id == -1)
{
cout << "invalid query because the pre is not found: " << pre << endl;
return false;
}
int sub_id = -1;
int obj_id = -1;
// -1 if not found, this means this subject is a constant
map<string, int>::iterator _find_sub_itr = (this->var_str2id).find(sub);
if(_find_sub_itr != this->var_str2id.end())
@ -555,12 +592,110 @@ BasicQuery::encodeBasicQuery(KVstore* _p_kvstore, const vector<string>& _query_v
return true;
}
int
BasicQuery::getSelectedVarPosition(int _var)
{
map<int, int>::iterator it = this->selected_var_position.find(_var);
if(it == this->selected_var_position.end())
{
return -1;
}
else
{
return it->second;
}
}
int
BasicQuery::getSelectedPreVarPosition(int _var)
{
//change var to -1-var to avoid conflicting with normal var
map<int, int>::iterator it = this->selected_var_position.find(-1-_var);
if(it == this->selected_var_position.end())
{
return -1;
}
else
{
return it->second;
}
}
int
BasicQuery::getSelectedVarPosition(string _var)
{
int id = this->getIDByVarName(_var);
if(id < 0)
{
return -1;
}
map<int, int>::iterator it = this->selected_var_position.find(id);
if(it == this->selected_var_position.end())
{
return -1;
}
else
{
return it->second;
}
}
int
BasicQuery::getSelectedPreVarPosition(string _var)
{
int id = this->getPreVarID(_var);
if(id < 0)
{
return -1;
}
//change var to -1-var to avoid conflicting with normal var
map<int, int>::iterator it = this->selected_var_position.find(-1-id);
if(it == this->selected_var_position.end())
{
return -1;
}
else
{
return it->second;
}
}
bool
BasicQuery::getEncodeBasicQueryResult() const
{
return this->encode_result;
}
int
BasicQuery::getIDByVarName(const string& _name)
{
map<string,int>::iterator it = this->var_str2id.find(_name);
if(it != this->var_str2id.end())
{
return it->second;
}
else
{
//WARN:if return var_str2id[_name] directly, this will be 0
//maybe conflict with valid ID 0
return -1;
}
}
bool
BasicQuery::isVarSelected(const string& _name) const
{
set<string>::iterator it = this->selected_var_set.find(_name);
if(it == this->selected_var_set.end())
{
return false;
}
else
{
return true;
}
}
int
BasicQuery::getPreVarID(const string& _name) const
{
@ -589,27 +724,17 @@ BasicQuery::getPreVarByID(unsigned _id) const
else
return this->pre_var[0];
}
//int
//BasicQuery::getIDByPreVarName(const string& _name) const
//{
//return 0;
//}
int
BasicQuery::getIDByVarName(const string& _name)
unsigned
BasicQuery::getSelectedPreVarNum() const
{
map<string,int>::iterator it = this->var_str2id.find(_name);
if(it != this->var_str2id.end())
{
return it->second;
}
else
{
//WARN:if return var_str2id[_name] directly, this will be 0
//maybe conflict with valid ID 0
return -1;
}
return this->selected_pre_var_num;
}
bool
BasicQuery::isPreVarSelected(unsigned _pid) const
{
//BETTER:check if the pid is valid
return this->pre_var[_pid].selected;
}
void
@ -628,7 +753,6 @@ BasicQuery::getTriple(int _i_th_triple)
void
BasicQuery::null_initial()
{
this->option_vs.clear();
this->triple_vt.clear();
this->var_str2id.clear();
this->var_degree = NULL;

26
Query/BasicQuery.h
View File

@ -102,17 +102,17 @@ public:
class BasicQuery
{
private:
vector<string> option_vs;
vector<Triple> triple_vt;
// mapping from variables' name to their assigned id
map<std::string, int> var_str2id;
// record each tuple's(subject, predicate, object) number of occurrences in this BasicQuery
map<std::string, int> tuple2freq;
set<std::string> selected_var_set;
//NOTICE:this is not used now!
map<std::string, int> var_not_in_select;
// id < select_var_num means in select
int select_var_num;
// var_num is different from that in SPARQLquery
// because there are some variable not in select
int graph_var_num;
@ -165,6 +165,12 @@ private:
//infos for predicate variables
vector<PreVar> pre_var;
int selected_pre_var_num;
//to manage the selected vars order(considering selected pre var)
//NOTICE:the order is always same as query_var in encodeBasicQuery, but not always same as select-clause
//positive key id for normal var, -1-id for pre var
map<int, int> selected_var_position;
public:
static const int MAX_VAR_NUM = 10;
@ -192,14 +198,16 @@ public:
// get the name of _var in the query graph
std::string getVarName(int _var);
int getIDByVarName(const string& _name);
// get triples number, also sentences number
int getTripleNum();
int getIDByVarName(const string& _name);
//check if a normal var is in select
bool isVarSelected(const std::string& _name) const;
std::string to_str();
// get the ID of the i-th triple
const Triple& getTriple(int _i_th_triple);
@ -219,7 +227,7 @@ public:
int getVarDegree(int _var);
//get the index of edge between two var ids
int getEdgeIndex(int _id0, int _id);
vector<int> getEdgeIndex(int _id0, int _id);
const EntityBitSet& getVarBitSet(int _i)const;
@ -269,8 +277,14 @@ public:
unsigned getPreVarNum() const;
const PreVar& getPreVarByID(unsigned) const;
//int getIDByPreVarName(const std::string& _name) const;
int getPreVarID(const string& _name) const;
unsigned getSelectedPreVarNum() const;
bool isPreVarSelected(unsigned _pid) const;
int getSelectedVarPosition(int _var);
int getSelectedPreVarPosition(int _var);
int getSelectedVarPosition(std::string _var);
int getSelectedPreVarPosition(std::string _var);
void addTriple(const Triple& _triple);
void print(ostream& _out_stream);

26
Query/GeneralEvaluation.cpp
View File

@ -28,7 +28,7 @@ bool GeneralEvaluation::parseQuery(const string &_query)
}
catch(const char *e)
{
cerr << e << endl;
cout << e << endl;
return false;
}
@ -40,25 +40,25 @@ QueryTree& GeneralEvaluation::getQueryTree()
return this->query_tree;
}
void GeneralEvaluation::doQuery()
bool GeneralEvaluation::doQuery()
{
if (!this->query_tree.checkProjectionAsterisk() && this->query_tree.getProjection().size() == 0)
return;
return false;
if (!this->query_tree.checkSelectCompatibility())
{
cerr << "[ERROR] The vars and aggregate functions in the SelectClause are not compatible." << endl;
return;
cout << "[ERROR] The vars and aggregate functions in the SelectClause are not compatible." << endl;
return false;
}
this->query_tree.getGroupPattern().getVarset();
if (this->query_tree.getGroupPattern().grouppattern_subject_object_maximal_varset.hasCommonVar(this->query_tree.getGroupPattern().grouppattern_predicate_maximal_varset))
{
cerr << "[ERROR] There are some vars occur both in subject/object and predicate." << endl;
return;
cout << "[ERROR] There are some vars occur both in subject/object and predicate." << endl;
return false;
}
this->strategy = Strategy(this->kvstore, this->vstree, this->pre2num);
this->strategy = Strategy(this->kvstore, this->vstree, this->pre2num, this->limitID_predicate, this->limitID_literal);
if (this->query_tree.checkWellDesigned())
{
cout << "=================" << endl;
@ -91,6 +91,8 @@ void GeneralEvaluation::doQuery()
long tv_postproc = Util::get_cur_time();
cout << "after Postprocessing, used " << (tv_postproc - tv_handle) << "ms." << endl;
}
return true;
}
void GeneralEvaluation::getBasicQuery(QueryTree::GroupPattern &grouppattern)
@ -2535,6 +2537,8 @@ void GeneralEvaluation::prepareUpdateTriple(QueryTree::GroupPattern &update_patt
object_id = Varset(update_pattern.patterns[i].object.value).mapTo(results_id->results[j].var)[0];
string subject, predicate, object;
TripleWithObjType::ObjectType object_type;
if (subject_id == -1)
subject = update_pattern.patterns[i].subject.value;
if (predicate_id == -1)
@ -2550,8 +2554,12 @@ void GeneralEvaluation::prepareUpdateTriple(QueryTree::GroupPattern &update_patt
this->stringindex->randomAccess(results_id->results[j].res[k][predicate_id], &predicate, false);
if (object_id != -1)
this->stringindex->randomAccess(results_id->results[j].res[k][object_id], &object, true);
if (object[0] == '<')
object_type = TripleWithObjType::Entity;
else
object_type = TripleWithObjType::Literal;
update_triple[update_triple_count++] = TripleWithObjType(subject, predicate, object);
update_triple[update_triple_count++] = TripleWithObjType(subject, predicate, object, object_type);
}
}
}

8
Query/GeneralEvaluation.h
View File

@ -35,14 +35,16 @@ class GeneralEvaluation
VSTree *vstree;
KVstore *kvstore;
TNUM* pre2num;
int limitID_predicate;
int limitID_literal;
StringIndex *stringindex;
Strategy strategy;
ResultFilter result_filter;
bool need_output_answer;
public:
explicit GeneralEvaluation(VSTree *_vstree, KVstore *_kvstore, StringIndex *_stringindex, TNUM* _pre2num):
vstree(_vstree), kvstore(_kvstore), stringindex(_stringindex), pre2num(_pre2num), need_output_answer(false)
explicit GeneralEvaluation(VSTree *_vstree, KVstore *_kvstore, StringIndex *_stringindex, TNUM* _pre2num, int _limitID_predicate, int _limitID_literal):
vstree(_vstree), kvstore(_kvstore), stringindex(_stringindex), pre2num(_pre2num), limitID_predicate(_limitID_predicate), limitID_literal(_limitID_literal), need_output_answer(false)
{
}
@ -51,7 +53,7 @@ class GeneralEvaluation
bool parseQuery(const std::string &_query);
QueryTree& getQueryTree();
void doQuery();
bool doQuery();
void getBasicQuery(QueryTree::GroupPattern &grouppattern);

4
Query/IDList.cpp
View File

@ -216,7 +216,7 @@ IDList::intersectList(const int* _id_list, int _list_len)
break;
}
default:
cerr << "no such method in IDList::intersectList()" << endl;
cout << "no such method in IDList::intersectList()" << endl;
break;
}
@ -450,7 +450,7 @@ IDList::intersect(const IDList& _id_list, const int* _list, int _len)
break;
}
default:
cerr << "no such method in IDList::intersectList()" << endl;
cout << "no such method in IDList::intersectList()" << endl;
break;
}

2
README.md
View File

@ -16,7 +16,7 @@ This system is really user-friendly and you can pick it up in several minutes. R
- type `git clone git@github.com:Caesar11/gStore.git` in your terminal or use git GUI to acquire it
Then you need to compile the project, just type `make` in the gStore root directory, and all executables will be ok. To run gStore, please type `bin/gload database_name dataset_path` to build a database named by yourself. And you can use `bin/gquery database_name` command to query a existing database. What is more, `bin/gconsole` is a wonderful tool designed for you, providing all operations you need to use gStore. Notice that all commands should be typed in the root directory of gStore, and your database name should not end with ".db".
Then you need to compile the project, just type `make` in the gStore root directory, and all executables will be ok. To run gStore, please type `bin/gbuild database_name dataset_path` to build a database named by yourself. And you can use `bin/gquery database_name` command to query a existing database. What is more, `bin/gconsole` is a wonderful tool designed for you, providing all operations you need to use gStore. Notice that all commands should be typed in the root directory of gStore, and your database name should not end with ".db".
- - -

2
Server/Client.cpp
View File

@ -232,7 +232,6 @@ void Client::run()
{
cmd = string(buf);
}
//DEBUG!
printf("%s\n", cmd.c_str());
free(buf);
@ -262,6 +261,7 @@ void Client::run()
string recv_msg;
flag = this->receiveResponse(recv_msg);
//BETTER:the string maybe too large?
fprintf(fp, "%s\n", recv_msg.c_str());
this->disconnectServer();

102
Server/Server.cpp
View File

@ -165,13 +165,13 @@ Server::listen()
}
break;
}
case CMD_INSERT:
{
string db_name = operation.getParameter(0);
string rdf_path = operation.getParameter(1);
this->insertTriple(db_name, "", rdf_path, ret_msg);
break;
}
//case CMD_INSERT:
//{
//string db_name = operation.getParameter(0);
//string rdf_path = operation.getParameter(1);
//this->insertTriple(db_name, "", rdf_path, ret_msg);
//break;
//}
case CMD_STOP:
{
this->stopServer(ret_msg);
@ -355,7 +355,15 @@ Server::dropDatabase(std::string _db_name, std::string _ac_name, std::string& _r
bool
Server::loadDatabase(std::string _db_name, std::string _ac_name, std::string& _ret_msg)
{
this->database = new Database(_db_name);
if(this->database == NULL)
{
this->database = new Database(_db_name);
}
else
{
_ret_msg = "please unload the current db first: " + this->database->getName();
return false;
}
bool flag = this->database->load();
@ -416,29 +424,29 @@ Server::importRDF(std::string _db_name, std::string _ac_name, std::string _rdf_p
return flag;
}
bool
Server::insertTriple(std::string _db_name, std::string _ac_name, std::string _rdf_path, std::string& _ret_msg)
{
if (this->database != NULL)
{
this->database->unload();
delete this->database;
}
//bool
//Server::insertTriple(std::string _db_name, std::string _ac_name, std::string _rdf_path, std::string& _ret_msg)
//{
//if (this->database != NULL)
//{
//this->database->unload();
//delete this->database;
//}
this->database = new Database(_db_name);
bool flag = this->database->insert(_rdf_path);
//this->database = new Database(_db_name);
//bool flag = this->database->insert(_rdf_path);
if (flag)
{
_ret_msg = "insert triple file to database done.";
}
else
{
_ret_msg = "import triple file to database failed.";
}
//if (flag)
//{
//_ret_msg = "insert triple file to database done.";
//}
//else
//{
//_ret_msg = "import triple file to database failed.";
//}
return flag;
}
//return flag;
//}
bool
Server::query(const string _query, string& _ret_msg)
@ -452,25 +460,47 @@ Server::query(const string _query, string& _ret_msg)
return false;
}
FILE* output = stdout;
FILE* output = NULL;
string path = "logs/gserver_query.log";
#ifdef OUTPUT_QUERY_RESULT
output = fopen(path.c_str(), "w");
#endif
ResultSet res_set;
bool flag = this->database->query(_query, res_set, output);
if (output != stdout) {
int query_ret = this->database->query(_query, res_set, output);
if (output != NULL)
{
fclose(output);
output = stdout;
}
if (flag)
bool flag = true;
//cout<<"Server query ret: "<<query_ret<<endl;
if (query_ret <= -100) //select query
{
//_ret_msg = "results are too large!";
//BETTER: divide and transfer if too large to be placed in memory, using Stream
_ret_msg = res_set.to_str();
if(query_ret == -100)
{
_ret_msg = res_set.to_str();
}
else //query error
{
flag = false;
_ret_msg = "query failed.";
//BETTER: {type:select} {success:false}
}
}
else
else //update query
{
_ret_msg = "query failed.";
if(query_ret >= 0)
{
_ret_msg = "update num: " + Util::int2string(query_ret);
}
else //update error
{
flag = false;
_ret_msg = "update failed.";
}
}
return flag;

2
Server/Server.h
View File

@ -48,7 +48,7 @@ public:
bool unloadDatabase(std::string _db_name, std::string _ac_name, std::string& _ret_msg);
bool showDatabases(std::string _para, std::string _ac_name, std::string& _ret_msg);
bool importRDF(std::string _db_name, std::string _ac_name, std::string _rdf_path, std::string& _ret_msg);
bool insertTriple(std::string _db_name, std::string _ac_name, std::string _rdf_path, std::string& _ret_msg);
//bool insertTriple(std::string _db_name, std::string _ac_name, std::string _rdf_path, std::string& _ret_msg);
bool query(const std::string _query, std::string& _ret_msg);
bool stopServer(std::string& _ret_msg);

12
Util/Util.cpp
View File

@ -247,7 +247,7 @@ Util::Util()
this->debug_kvstore = fopen(s.c_str(), "w+");
if(this->debug_kvstore == NULL)
{
printf("open error: kv.log\n");
cerr << "open error: kv.log\n";
this->debug_kvstore = stderr;
}
}
@ -259,7 +259,7 @@ Util::Util()
this->debug_database = fopen(s.c_str(), "w+");
if(this->debug_database == NULL)
{
printf("open error: db.log\n");
cerr << "open error: db.log\n";
this->debug_database = stderr;
}
}
@ -271,7 +271,7 @@ Util::Util()
this->debug_vstree = fopen(s.c_str(), "w+");
if(this->debug_vstree == NULL)
{
printf("open error: vs.log\n");
cerr << "open error: vs.log\n";
this->debug_vstree = stderr;
}
}
@ -660,7 +660,7 @@ string
Util::getQueryFromFile(const char* _file_path)
{
#ifdef DEBUG_PRECISE
printf("file to open: %s\n", _file_path);
cerr << "file to open: " << _file_path <<endl;
#endif
char buf[10000];
std::string query_file;
@ -775,7 +775,7 @@ Util::getSystemOutput(string cmd)
//fclose(fp);
//}
system(cmd.c_str());
cout<<"ans: "<<ans<<endl;
//cerr<<"ans: "<<ans<<endl;
return ans;
}
@ -1321,4 +1321,4 @@ Util::node2string(const char* _raw_str) {
_output += _raw_str[i];
}
return _output;
}
}

8
Util/Util.h
View File

@ -67,6 +67,8 @@ in the sparql query can point to the same node in data graph)
#include <readline/history.h>
#define STREAM_ON 1
//when used as C/S, if output query result in the server port: default not(you can see the result in the client)
//#define OUTPUT_QUERY_RESULT 1
#define READLINE_ON 1
#define MULTI_INDEX 1
//#define SO2P 1
@ -74,12 +76,14 @@ in the sparql query can point to the same node in data graph)
//#define USE_GROUP_DELETE 1
//indicate that in debug mode
#define DEBUG_JOIN
//#define DEBUG_STREAM
//#define DEBUG_PRECISE 1 all information
//#define DEBUG_KVSTORE 1 //in KVstore
//#define DEBUG_VSTREE 1 //in Database
#define DEBUG_VSTREE 1 //in Database
//#define DEBUG_DATABASE 1 //in Database
//#define DEBUG_JOIN
//
//
#ifdef DEBUG_PRECISE
#ifndef DEBUG

8
VSTree/LRUCache.cpp
View File

@ -44,14 +44,14 @@ LRUCache::LRUCache(int _capacity)
LRUCache::~LRUCache()
{
delete[]this->next;
delete[]this->prev;
delete[]this->keys;
delete[] this->next;
delete[] this->prev;
delete[] this->keys;
for (int i = 0; i < this->size; ++i)
{
delete this->values[i];
}
delete[]this->values;
delete[] this->values;
}
//load cache's elements from an exist data file.

161
VSTree/VSTree.cpp
View File

@ -377,7 +377,7 @@ VSTree::insertEntry(const SigEntry& _entry)
VNode* choosedNodePtr = this->chooseNode(this->getRoot(), _entry);
#ifdef DEBUG_VSTREE
if(_entry.getEntityId() == 2402)
if(_entry.getEntityId() == 200)
{
stringstream _ss;
if (choosedNodePtr)
@ -402,6 +402,9 @@ VSTree::insertEntry(const SigEntry& _entry)
if (choosedNodePtr->isFull())
{
#ifdef DEBUG_VSTREE
cout<<"split occur"<<endl;
#endif
//if the choosed leaf node to insert is full, the node should be split.
this->split(choosedNodePtr, _entry, NULL);
@ -433,6 +436,10 @@ VSTree::insertEntry(const SigEntry& _entry)
}
this->entry_num ++;
#ifdef DEBUG_VSTREE
//cout<<"file line check: "<<this->entityID2FileLineMap[200]<<endl;
#endif
return true;
}
@ -440,6 +447,7 @@ VSTree::insertEntry(const SigEntry& _entry)
bool
VSTree::removeEntry(int _entity_id)
{
//cout<<"file line check: "<<this->entityID2FileLineMap[200]<<endl;
VNode* leafNodePtr = this->getLeafNodeByEntityID(_entity_id);
if (leafNodePtr == NULL)
@ -478,6 +486,9 @@ VSTree::removeEntry(int _entity_id)
leafNodePtr->removeChild(entryIndex);
leafNodePtr->refreshAncestorSignature(*(this->node_buffer));
this->removeNode(leafNodePtr);
//DEBUG: already deleted in freeElem
//delete leafNodePtr;
//leafNodePtr = NULL;
this->root_file_line = -1;
this->height = 0;
this->entry_num = 0;
@ -485,6 +496,7 @@ VSTree::removeEntry(int _entity_id)
}
else
{
//cout<<"root remove a child"<<endl;
leafNodePtr->removeChild(entryIndex);
leafNodePtr->refreshAncestorSignature(*(this->node_buffer));
}
@ -718,57 +730,86 @@ VSTree::split(VNode* _p_node_being_split, const SigEntry& _insert_entry, VNode*
entryIndex_nearB.clear();
entryIndex_nearA.push_back(entryA_index);
entryIndex_nearB.push_back(entryB_index);
int cnt = 1, i;
//BETTER:maybe sort and add?(how to sort according to two seeds)
for(i = 0; i < VNode::MAX_CHILD_NUM; ++i)
{
if(i == entryA_index || i == entryB_index)
{
continue;
}
if(cnt > VNode::MIN_CHILD_NUM) //num+1
{
break;
}
cnt++;
entryIndex_nearA.push_back(i);
}
for(; i < VNode::MAX_CHILD_NUM; ++i)
{
if(i == entryA_index || i == entryB_index)
{
continue;
}
entryIndex_nearB.push_back(i);
}
int nearA_max_size, nearB_max_size;
bool nearA_tooSmall, nearB_tooSmall;
//NOTICE: code below maybe exist error, can not divide evenly(and maybe not necessary to compute distance)
//
//int nearA_max_size, nearB_max_size;
//bool nearA_tooSmall, nearB_tooSmall;
//for(int i = 0; i < VNode::MAX_CHILD_NUM; i++)
//{
//if(i == entryA_index || i == entryB_index) continue;
for(int i = 0; i < VNode::MAX_CHILD_NUM; i++)
{
if(i == entryA_index || i == entryB_index) continue;
////should guarantee that each new node has at least MIN_CHILD_NUM children.
//nearA_max_size = VNode::MAX_CHILD_NUM - entryIndex_nearB.size();
//nearA_tooSmall = (nearA_max_size < VNode::MIN_CHILD_NUM);
//should guarantee that each new node has at least MIN_CHILD_NUM children.
nearA_max_size = VNode::MAX_CHILD_NUM - entryIndex_nearB.size();
nearA_tooSmall = (nearA_max_size <= VNode::MIN_CHILD_NUM);
////BETTER:is this too wasteful?
//if(nearA_tooSmall)
//{
//for(; i < VNode::MAX_CHILD_NUM; i++)
//{
//if (i == entryA_index || i == entryB_index) continue;
//entryIndex_nearA.push_back(i);
//}
//break;
//}
if(nearA_tooSmall)
{
for(; i < VNode::MAX_CHILD_NUM; i++)
{
if (i == entryA_index || i == entryB_index) continue;
entryIndex_nearA.push_back(i);
}
break;
}
//nearB_max_size = VNode::MAX_CHILD_NUM - entryIndex_nearA.size();
//nearB_tooSmall = (nearB_max_size < VNode::MIN_CHILD_NUM);
//if(nearB_tooSmall)
//{
//for(; i < VNode::MAX_CHILD_NUM; i++)
//{
//if(i == entryA_index || i == entryB_index) continue;
//entryIndex_nearB.push_back(i);
//}
//break;
//}
nearB_max_size = VNode::MAX_CHILD_NUM - entryIndex_nearA.size();
nearB_tooSmall = (nearB_max_size <= VNode::MIN_CHILD_NUM);
if(nearB_tooSmall)
{
for(; i < VNode::MAX_CHILD_NUM; i++)
{
if(i == entryA_index || i == entryB_index) continue;
entryIndex_nearB.push_back(i);
}
break;
}
////calculate the distance from
////the i-th child entry signature to seedA(or seedB).
//calculate the distance from
//the i-th child entry signature to seedA(or seedB).
////NOTICE:we should expect that the candidate can be almost contained!
////However, the precondition there are not too many 1s
//int disToSeedA = entryA.xEpsilen(_p_node_being_split->getChildEntry(i));
//int disToSeedB = entryB.xEpsilen(_p_node_being_split->getChildEntry(i));
//// choose the near one seed to add into
//if(disToSeedA <= disToSeedB)
//{
//entryIndex_nearA.push_back(i);
//}
//else
//{
//entryIndex_nearB.push_back(i);
//}
//}
//NOTICE:we should expect that the candidate can be almost contained!
//However, the precondition there are not too many 1s
int disToSeedA = entryA.xEpsilen(_p_node_being_split->getChildEntry(i));
int disToSeedB = entryB.xEpsilen(_p_node_being_split->getChildEntry(i));
// choose the near one seed to add into
if(disToSeedA <= disToSeedB)
{
entryIndex_nearA.push_back(i);
}
else
{
entryIndex_nearB.push_back(i);
}
}
#ifdef DEBUG_VSTREE
cout<<"A: "<<entryIndex_nearA.size()<<" B: "<<entryIndex_nearB.size()<<endl;
#endif
// then create a new node to act as BEntryIndex's father.
VNode* newNodePtr = this->createNode();
@ -795,7 +836,6 @@ VSTree::split(VNode* _p_node_being_split, const SigEntry& _insert_entry, VNode*
}
else
{
//debug target 2
VNode* childPtr = oldNodePtr->getChild(entryIndex_nearB[i], *(this->node_buffer));
newNodePtr->addChildNode(childPtr);
}
@ -869,8 +909,10 @@ VSTree::split(VNode* _p_node_being_split, const SigEntry& _insert_entry, VNode*
//should keep the root node always being
//at the first line(line zero) of the tree node file.
this->swapNodeFileLine(RootNewPtr, oldNodePtr);
cout<<"new root: "<<RootNewPtr->getFileLine()<<endl;
this->height++;
this->root_file_line = RootNewPtr->getFileLine();
//NOTICE:below is unnecessary
//this->root_file_line = RootNewPtr->getFileLine();
#ifdef DEBUG
cout<<"root file line "<<this->root_file_line<<" child num "<<RootNewPtr->getChildNum()<<endl;
@ -939,6 +981,7 @@ VSTree::coalesce(VNode* _child, int _entry_index)
if(_father == NULL) //this is already root
{
//cout<<"the father is NULL!!!"<<endl;
//NOTICE:when root is leaf, at least one key, otherwise the tree is empty
//But when root is internal, at least two key, if one key then shrink
//(1-key internal root is not permitted)
@ -951,18 +994,23 @@ VSTree::coalesce(VNode* _child, int _entry_index)
//only one key after remove, shrink root
VNode* newRoot = _child->getChild(0, *(this->node_buffer));
newRoot->setAsRoot(true);
#ifdef DEBUG
cout<<"shrink root in coalesce() -- to swap node file"<<endl;
#endif
this->swapNodeFileLine(newRoot, _child);
this->root_file_line = newRoot->getFileLine();
//this->root_file_line = newRoot->getFileLine();
this->height--;
this->removeNode(_child);
}
return;
}
//cout<<"num: "<<cn<<endl;
if(cn > VNode::MIN_CHILD_NUM)
{
#ifdef DEBUG_VSTREE
cout<<"no need to move or union in coalesce()"<<endl;
#endif
_child->removeChild(_entry_index);
_child->refreshAncestorSignature(*(this->node_buffer));
return;
@ -1158,6 +1206,12 @@ VSTree::coalesce(VNode* _child, int _entry_index)
{
this->updateEntityID2FileLineMap(p);
}
else
{
//DEBUG
//delete p;
//p = NULL;
}
}
}
@ -1356,10 +1410,12 @@ VSTree::loadTreeInfo()
fread(&key, sizeof(int), 1, filePtr);
while(!feof(filePtr))
{
//if(key == 25)
//{
//cout<<"loadTreeInfo() - get id 25"<<endl;
//}
#ifdef DEBUG_VSTREE
if(key == 1)
{
cout<<"loadTreeInfo() - get id 1"<<endl;
}
#endif
this->free_nid_list.push_back(key);
fread(&key, sizeof(int), 1, filePtr);
}
@ -1608,6 +1664,7 @@ VSTree::removeNode(VNode* _vp)
this->node_buffer->del(key);
this->node_num--;
//delete _vp;
//_vp->setFileLine(-1);
}
string

7
api/cpp/example/Makefile
View File

@ -1,8 +1,11 @@
#CC=g++
CC=ccache g++
example: CppAPIExample.o
g++ -o example CppAPIExample.o -L../lib -lgstoreconnector
$(CC) -o example CppAPIExample.o -L../lib -lgstoreconnector
CppAPIExample.o: CppAPIExample.cpp
g++ -c -I../src/ CppAPIExample.cpp -o CppAPIExample.o
$(CC) -c -I../src/ CppAPIExample.cpp -o CppAPIExample.o
clean:
rm -rf *.o example

5
api/cpp/src/Makefile
View File

@ -1,3 +1,6 @@
#CC=g++
CC=ccache g++
lib_dir=../lib/
socket_obj_dir=../../../.objs/
@ -7,7 +10,7 @@ $(lib_dir)libgstoreconnector.a: GstoreConnector.o $(socket_obj_dir)Socket.o
ar -crv $(lib_dir)libgstoreconnector.a GstoreConnector.o $(socket_obj_dir)Socket.o
GstoreConnector.o: GstoreConnector.cpp GstoreConnector.h $(socket_obj_dir)Socket.o
g++ -c GstoreConnector.cpp -o GstoreConnector.o
$(CC) -c GstoreConnector.cpp -o GstoreConnector.o
clean:
rm -rf GstoreConnector.o $(lib_dir)libgstoreconnector.a

6
data/all.sql
View File

@ -1,4 +1,4 @@
select ?s ?p ?o where
{
?s ?p ?o .
select ?s ?o where
{
?s ?p ?o .
}

4
data/lubm_pre.sql Normal file
View File

@ -0,0 +1,4 @@
select ?x ?p where
{
?x ?p <FullProfessor0>.
}

9
data/p1.sql Normal file
View File

@ -0,0 +1,9 @@
select distinct ?x ?p1 ?p2 where
{
?x <rdf:type> <ub:GraduateStudent>.
?y <rdf:type> <ub:University>.
?z <rdf:type> <ub:Department>.
?x ?p1 ?z.
?z ?p2 ?y.
?x <ub:undergraduateDegreeFrom> ?y.
}

10
data/p2.sql Normal file
View File

@ -0,0 +1,10 @@
select distinct ?x ?p1 ?p2 ?p3 where
{
?x <rdf:type> <ub:UndergraduateStudent>.
?y <ub:name> <Course1>.
?x ?p1 ?y.
?z <ub:teacherOf> ?y.
?z <ub:name> <FullProfessor1>.
?z ?p2 ?w.
?w ?p3 <Department0>.
}

4
data/p3.sql Normal file
View File

@ -0,0 +1,4 @@
select distinct ?x where
{
?x <rdf:type> <ub:UndergraduateStudent>.
}

4
data/para1.sql Normal file
View File

@ -0,0 +1,4 @@
select ?p where
{
<s0> ?p <o0> .
}

5
data/para2.sql Normal file
View File

@ -0,0 +1,5 @@
select ?s ?o where
{
?s <p0> ?o .
?s <p1> ?o .
}

5
data/pre1.sql Normal file
View File

@ -0,0 +1,5 @@
select ?s ?p ?o ?p2 ?o2 where
{
?s ?p ?o .
?o ?p2 ?o2 .
}

5
data/pre2.sql Normal file
View File

@ -0,0 +1,5 @@
select ?s ?p2 ?o2 where
{
?s ?p ?o .
?o ?p2 ?o2 .
}

6
data/pre3.sql Normal file
View File

@ -0,0 +1,6 @@
select ?s1 ?s2 ?p3 ?o where
{
?s1 ?p1 ?o .
?s2 ?p2 ?o .
?s3 ?p3 ?o .
}

7
data/pre4.sql Normal file
View File

@ -0,0 +1,7 @@
select ?s1 ?s2 ?p3 ?o where
{
?s1 ?p1 ?o .
?s2 ?p2 ?o .
?s3 ?p3 ?o .
?s1 ?p4 ?o2 .
}

4
data/small_all.sql
View File

@ -1,4 +0,0 @@
select ?s ?o where
{
?s ?p ?o .
}

6
data/small_p3.sql Normal file
View File

@ -0,0 +1,6 @@
select ?v2 ?v0 where
{
?v0 <own> ?v1 .
?v0 <own> ?v2 .
?v1 <close> ?v2 .
}

9
docs/DEMAND.md
View File

@ -15,6 +15,7 @@ readline-devel | need to be installed
openjdk | needed if using Java api
openjdk-devel | needed if using Java api
realpath | needed if using gconsole
ccache | optional, used to speed up the compilation
NOTICE:
@ -22,13 +23,15 @@ NOTICE:
2. To install readline and readline-devel, just type `dnf install readline-devel` in Redhat/CentOS/Fedora, or `apt-get install libreadline-dev` in Debian/Ubuntu. Please use corresponding commands in other systems. If you use ArchLinux, just type `pacman -S readline` to install the readline and readline-devel.(so do other packages)
3. You do not have to install realpath to use gStore, but if you want to use the gconsole for its convenience, please do so by using `dnf install realpath` or `apt-get install realpath`.
3. You do not have to install realpath to use gStore, but if you want to use the gconsole for its convenience, please do so by using `dnf install realpath` or `apt-get install realpath`. However, if you can not install realpath in your system, please go to modify the file Main/gconsole.cpp(just find the place using realpath command and remove the realpath command).
4. Our programs use regEx functions, which are provided by GNU/Linux by default. You do not need to have to install boost and boost-devel for more powerful regEx libraries.
4. Our programs use regEx functions, which are provided by GNU/Linux by default. You do not need to install boost and boost-devel for more powerful regEx libraries.
5. ANTLR3.4 is used in gStore to produce lexer and parser code for SPARQL query. However, you do not need to install the corresponding antlr libraries because we have merged the libantlr3.4 in our system.
6. When you type `make` in the root directory of the gStore project, the Java api will also be compiled. You can modify the makefile if you do not have JDK in your system. However, you are advised to install openjdk-devel in your Linux system.
7. Any other questions, please go to [FAQ](FAQ.md) page.
7. To install ccache, you need to add epel repository if using CentOS, while in Ubuntu you can directly install it by `apt-get install ccache` command. If you can not install ccache(or maybe you do not want to), please go to modify the makefile(just change the CC variable to g++).
8. Any other questions, please go to [FAQ](FAQ.md) page.

BIN
docs/docx/02-Gstore2.0_useCaseDoc.docx
View File

Binary file not shown.

BIN
docs/docx/05-gStore2.0-usage_eng.docx
View File

Binary file not shown.

BIN
docs/docx/Gstore2.0软件需求说明.docx
View File

Binary file not shown.

BIN
docs/latex/gStore_help.pdf
View File

Binary file not shown.

7
docs/latex/gStore_help.tex
View File

@ -253,6 +253,7 @@ readline-devel & need to be installed\tabularnewline
openjdk & needed if using Java api\tabularnewline
openjdk-devel & needed if using Java api\tabularnewline
realpath & needed if using gconsole\tabularnewline
ccache & optional, used to speed up the compilation\tabularnewline
\bottomrule
\caption{software requirement}
\end{longtable}
@ -265,13 +266,15 @@ NOTICE:
\item
To install readline and readline-devel, just type \texttt{dnf\ install\ readline-devel} in Redhat/CentOS/Fedora, or \texttt{apt-get\ install\ libreadline-dev} in Debian/Ubuntu. Please use corresponding commands in other systems. If you use ArchLinux, just type \texttt{pacman\ -S\ readline} to install the readline and readline-devel.(so do other packages)
\item
You do not have to install realpath to use gStore, but if you want to use the gconsole for its convenience, please do so by using \texttt{dnf\ install\ realpath} or \texttt{apt-get\ install\ realpath}.
You do not have to install realpath to use gStore, but if you want to use the gconsole for its convenience, please do so by using \texttt{dnf\ install\ realpath} or \texttt{apt-get\ install\ realpath}. However, if you can not install realpath in your system, please go to modify the file Main/gconsole.cpp(just find the place using realpath command and remove the realpath command).
\item
Our programs use regEx functions, which are provided by GNU/Linux by default. You do not need to have to install boost and boost-devel for more powerful regEx libraries.
Our programs use regEx functions, which are provided by GNU/Linux by default. You do not need to install boost and boost-devel for more powerful regEx libraries.
\item
ANTLR3.4 is used in gStore to produce lexer and parser code for SPARQL query. However, you do not need to install the corresponding antlr libraries because we have merged the libantlr3.4 in our system.
\item
When you type \texttt{make} in the root directory of the gStore project, the Java api will also be compiled. You can modify the makefile if you do not have JDK in your system. However, you are advised to install openjdk-devel in your Linux system.
\item
To install ccache, you need to add epel repository if using CentOS, while in Ubuntu you can directly install it by `apt-get install ccache` command. If you can not install ccache(or maybe you do not want to), please go to modify the makefile(just change the CC variable to g++).
\item
Any other questions, please go to \hyperref[chapter10]{FAQ} page.
\end{enumerate}

BIN
docs/latex/gStore_help_CN.pdf
View File

Binary file not shown.

5
docs/latex/gStore_help_CN.tex
View File

@ -394,6 +394,7 @@ Gstore系统（也称作gStore）是一个用于管理大型图结构数据的
openjdk & 使用Java api时需要 \\
openjdk-devel & 使用Java api时需要 \\
realpath & 使用gconsole时需要 \\
ccache & 可选, 可以加速编译过程 \\
\bottomrule
\end{tabular}
\caption{软件要求}
@ -407,7 +408,7 @@ Gstore系统（也称作gStore）是一个用于管理大型图结构数据的
\item
要安装readline和readline-devel，只需要在Redhat/CentOS/Fedora中输入\texttt{dnf\ install\ readline-devel}，或者在Debian/Ubuntu 中输入\texttt{apt-get\ install\ libreadline-dev}。请在其他系统中使用对应的指令。如果你使用的是ArchLinux，只要输入\texttt{pacman\ -S\ readline}就可以安装readline和readline-devel。（其他包也一样）
\item
使用gStore不需要安装realpath，但如果你想要使用gconsole，请输入\texttt{dnf\ install\ realpath}或\texttt{apt-get\ install\ realpath}进行安装。
使用gStore不需要安装realpath，但如果你想要使用gconsole，请输入\texttt{dnf\ install\ realpath}或\texttt{apt-get\ install\ realpath}进行安装。 然而，如果你不想安装realpath，请修改Main/gconsole.cpp文件(找到使用realpath的地方并移除realpath命令即可)。
\item
我们的项目使用了正则表达式，由GNU/Linux默认提供。要使用更强大的正则表达式库，你不需要安装boost和boost-devel。
\item
@ -415,6 +416,8 @@ Gstore系统（也称作gStore）是一个用于管理大型图结构数据的
\item
当你在gStore项目的根目录下输入\texttt{make}时，Java api也会编译。如果你的系统里没有JDK，你可以修改makefile。我们建议你在Linux系统中安装openjdk-devel。
\item
在CentOS系统上你需要添加epel源才能安装ccache，但在Ubuntu系统上你可以直接安装。如果你无法安装ccache(或者你不想安装)，请修改makefile文件(只需将CC变量改为g++即可)。
\item
其他问题请参阅\hyperref[chapter10]{【FAQ】}一章。
\end{enumerate}

30
makefile Executable file → Normal file
View File

@ -18,22 +18,29 @@
#(also include good comments norm)
#http://blog.csdn.net/u010740725/article/details/51387810
#NOTICE: to speed up the make process, use make -j4
#use -j8 or higher may cause error
#http://blog.csdn.net/cscrazybing/article/details/50789482
#http://blog.163.com/liuhonggaono1@126/blog/static/10497901201210254622141/
#TODO:the dependences are not complete!
#compile parameters
CC = g++
CC = ccache g++
#CC = g++
#the optimazition level of gcc/g++
#http://blog.csdn.net/hit_090420216/article/details/44900215
#NOTICE: -O2 is recommended, while -O3 is dangerous
#when developing, not use -O because it will disturb the normal
#routine. use it for test and release.
#CFLAGS = -c -Wall -g #-fprofile-arcs -ftest-coverage #-pg
#EXEFLAG = -g #-fprofile-arcs -ftest-coverage #-pg
CFLAGS = -c -Wall -g #-fprofile-arcs -ftest-coverage #-pg
EXEFLAG = -g #-fprofile-arcs -ftest-coverage #-pg
#-coverage
CFLAGS = -c -Wall -O2
EXEFLAG = -O2
#CFLAGS = -c -Wall -O2
#EXEFLAG = -O2
#add -lreadline -ltermcap if using readline or objs contain readline
library = -ltermcap -lreadline -L./lib -lantlr -lgcov
@ -84,12 +91,21 @@ objfile = $(kvstoreobj) $(vstreeobj) $(stringindexobj) $(parserobj) $(serverobj)
inc = -I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include
#auto generate dependencies
# http://blog.csdn.net/gmpy_tiger/article/details/51849474
# http://blog.csdn.net/jeffrey0000/article/details/12421317
#gtest
all: $(exedir)gbuild $(exedir)gserver $(exedir)gclient $(exedir)gquery $(exedir)gconsole $(api_java) $(exedir)gadd $(exedir)gsub
TARGET = $(exedir)gbuild $(exedir)gserver $(exedir)gclient $(exedir)gquery $(exedir)gconsole $(api_java) $(exedir)gadd $(exedir)gsub
all: $(TARGET)
test_index: test_index.cpp
$(CC) $(EXEFLAG) -o test_index test_index.cpp $(objfile) $(library)
#BETTER: use for loop to reduce the lines
#NOTICE: g++ -MM will run error if linking failed, like Database.h/../SparlParser.h/../antlr3.h
#executables begin
#NOTICE:not include g*.o in objfile due to multiple definitions of main()
@ -374,10 +390,12 @@ clean:
rm -rf $(exedir)g* $(objdir)*.o $(exedir).gserver*
#rm -rf .project .cproject .settings just for eclipse
#rm -rf cscope* just for vim
rm -rf logs/*.log
dist: clean
rm -rf *.nt *.n3 .debug/*.log .tmp/*.dat *.txt *.db
rm -rf tools/libantlr3c-3.4 lib/libantlr.a Parser/Sparql*
#rm -rf Parser/SparqlLexer* Parser/SparlParser.cpp
rm -rf cscope* .cproject .settings tags
rm -rf *.info

286
test/auto_makefile Executable file
View File

@ -0,0 +1,286 @@
#help for make
#http://www.cnblogs.com/wang_yb/p/3990952.html
#https://segmentfault.com/a/1190000000349917
#http://blog.csdn.net/cuiyifang/article/details/7910268
#to use gprof to analyse efficience of the program:
#http://blog.chinaunix.net/uid-25194149-id-3215487.html
#to use gcov and lcov
#Notice that optimization should not be used here
#http://blog.163.com/bobile45@126/blog/static/96061992201382025729313/
#gcov -a main.cpp
#lcov --directory . --capture --output-file dig.info
#genhtml --output-directory . --frames --show-details dig.info
#to use doxygen+graphviz+htmlhelp to generate document from source code:
#http://www.ooxygen.nl/
#(also include good comments norm)
#http://blog.csdn.net/u010740725/article/details/51387810
#NOTICE: to speed up the make process, use make -j4
#use -j8 or higher may cause error
#http://blog.csdn.net/cscrazybing/article/details/50789482
#http://blog.163.com/liuhonggaono1@126/blog/static/10497901201210254622141/
#TODO:the dependences are not complete!
#compile parameters
CC = ccache g++
#CC = g++
#the optimazition level of gcc/g++
#http://blog.csdn.net/hit_090420216/article/details/44900215
#NOTICE: -O2 is recommended, while -O3 is dangerous
#when developing, not use -O because it will disturb the normal
#routine. use it for test and release.
#CFLAGS = -c -Wall -g #-fprofile-arcs -ftest-coverage #-pg
#EXEFLAG = -g #-fprofile-arcs -ftest-coverage #-pg
#-coverage
CFLAGS = -c -Wall -O2
EXEFLAG = -O2
#add -lreadline -ltermcap if using readline or objs contain readline
library = -ltermcap -lreadline -L./lib -lantlr -lgcov
def64IO = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE
# paths
objdir = .objs/
exedir = bin/
lib_antlr = lib/libantlr.a
api_cpp = api/cpp/lib/libgstoreconnector.a
api_java = api/java/lib/GstoreJavaAPI.jar
inc = -I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include
#gtest
TARGET = $(exedir)gbuild $(exedir)gserver $(exedir)gclient $(exedir)gquery $(exedir)gconsole $(api_java) $(exedir)gadd $(exedir)gsub
objfile = $(objdir)SPARQLquery.o $(objdir)Database.o $(objdir)DBparser.o $(objdir)SparlParser.o $(objdir)SparqlLexer.o \
$(objdir)QueryParser.o $(objdir)Server.o $(objdir)GeneralEvaluation.o
#auto generate dependencies
# http://blog.csdn.net/gmpy_tiger/article/details/51849474
# http://blog.csdn.net/jeffrey0000/article/details/12421317
DEPEND = Util/Util.d Util/Bstr.d Util/Stream.d Util/Stream.d Util/Triple.d Util/BloomFilter.d \
KVstore/SITree/SITree.d KVstore/SITree/SIStorage.d KVstore/SITree/SINode.d KVstore/SITree/SIIntlNode.d KVstore/SITree/SILeafNode.d KVstore/SITree/SIHeap.d \
KVstore/ISTree/ISTree.d KVstore/ISTree/ISStorage.d KVstore/ISTree/ISNode.d KVstore/ISTree/ISIntlNode.d KVstore/ISTree/ISLeafNode.d KVstore/ISTree/ISHeap.d \
KVstore/KVstore.d Query/BasicQuery.d Query/ResultSet.d Query/IDList.d Database/Join.d Database/Strategy.d \
Query/Varset.d Query/QueryTree.d Query/ResultFilter.d StringIndex/StringIndex.d \
Signature/SigEntry.d Signature/Signature.d VSTree/VSTree.d VSTree/EntryBuffer.d VSTree/LRUCache.d VSTree/VNode.d \
Parser/RDFParser.d Parser/TurtleParser.d Server/Operation.d Server/Client.d Server/Socket.d
#NOTICE: not place tab before include
#-include Util/Util.d
#sinclude?
all: $(TARGET)
-include $(DEPEND)
test_index: test_index.cpp
$(CC) $(EXEFLAG) -o test_index test_index.cpp $(objfile) $(library)
#executables begin
#BETTER: use for loop to reduce the lines
#NOTICE: g++ -MM will run error if linking failed, like Database.h/../SparlParser.h/../antlr3.h
#NOTICE:not include g*.o in objfile due to multiple definitions of main()
$(exedir)gbuild: $(lib_antlr) $(objdir)gbuild.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gbuild $(objdir)gbuild.o $(objfile) $(library)
$(exedir)gquery: $(lib_antlr) $(objdir)gquery.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gquery $(objdir)gquery.o $(objfile) $(library)
$(exedir)gserver: $(lib_antlr) $(objdir)gserver.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gserver $(objdir)gserver.o $(objfile) $(library)
$(exedir)gclient: $(lib_antlr) $(objdir)gclient.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gclient $(objdir)gclient.o $(objfile) $(library)
$(exedir)gconsole: $(lib_antlr) $(objdir)gconsole.o $(objfile) $(api_cpp)
$(CC) $(EXEFLAG) -o $(exedir)gconsole $(objdir)gconsole.o $(objfile) $(library) -L./api/cpp/lib -lgstoreconnector
#executables end
#objects in Main/ begin
$(objdir)gbuild.o: Main/gbuild.cpp Database/Database.h Util/Util.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gbuild.cpp $(inc) -o $(objdir)gbuild.o
$(objdir)gquery.o: Main/gquery.cpp Database/Database.h Util/Util.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gquery.cpp $(inc) -o $(objdir)gquery.o #-DREADLINE_ON
#add -DREADLINE_ON if using readline
$(objdir)gserver.o: Main/gserver.cpp Server/Server.h Util/Util.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gserver.cpp $(inc) -o $(objdir)gserver.o
$(objdir)gclient.o: Main/gclient.cpp Server/Client.h Util/Util.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gclient.cpp $(inc) -o $(objdir)gclient.o #-DREADLINE_ON
$(objdir)gconsole.o: Main/gconsole.cpp Database/Database.h Util/Util.h api/cpp/src/GstoreConnector.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gconsole.cpp $(inc) -o $(objdir)gconsole.o -I./api/cpp/src/ #-DREADLINE_ON
#objects in Main/ end
# auto generated objects
#TODO+DEBUG:
#TODO:maybe multiple lines? how about auto compile with given CC/CFLAGS args? \\n
%.d: %.cpp
$(CC) -MM $^ > $@.tmp2
sed "s/\(\\\n\)/ /g" < $@.tmp2 > $@.tmp
#sed "s/\([^\.]*\)\(\.o: \)\([^\/]*\)/.objs\/\1.o \3\/\1.d: \3/g" < $@.tmp > $@
sed "s/\([^\.]*\)\(\.o: \)\([^\/]*\)\(.*\)/.objs\/\1.o: \3\4\n\t$(CC) $(CFLAGS) \3\/\1.cpp -o .objs\/\1.o/g" < $@.tmp > $@
#tmp=$^
#dir=${tmp%/*}
#tmp2=${tmp#*/}
#name=${tmp2%.*}
#NOTICE: use "" instead of '' if need to use shell var
#sed "s/\(${name}.o\)/$(objdir)\1 $@/g" < $@.tmp > $@
rm $@.tmp*
# special objects
$(objdir)Database.o: Database/Database.cpp Database/Database.h \
Util/Util.h Util/Triple.h Parser/DBparser.h Parser/SparlParser.h Parser/RDFParser.h \
Query/IDList.h Query/ResultSet.h Query/SPARQLquery.h Query/BasicQuery.h \
Signature/SigEntry.h KVstore/KVstore.h VSTree/VSTree.h
$(CC) $(CFLAGS) Database/Database.cpp $(inc) -o $(objdir)Database.o
$(objdir)DBparser.o: Parser/DBparser.cpp Parser/DBparser.h $(objdir)SparqlParser.o $(objdir)SparqlLexer.o $(objdir)Triple.o $(objdir)SPARQLquery.o
$(CC) $(CFLAGS) Parser/DBparser.cpp $(inc) -o $(objdir)DBparser.o
$(objdir)SparqlParser.o: Parser/SparqlParser.c Parser/SparqlParser.h
gcc $(CFLAGS) Parser/SparqlParser.c $(inc) -o $(objdir)SparqlParser.o
$(objdir)SparqlLexer.o: Parser/SparqlLexer.c Parser/SparqlLexer.h
gcc $(CFLAGS) Parser/SparqlLexer.c $(inc) -o $(objdir)SparqlLexer.o
$(objdir)QueryParser.o: Parser/QueryParser.cpp Parser/QueryParser.h $(objdir)SparqlParser.o $(objdir)SparqlLexer.o $(objdir)QueryTree.o $(objdir)Util.o
$(CC) $(CFLAGS) Parser/QueryParser.cpp $(inc) -o $(objdir)QueryParser.o
$(objdir)Server.o: Server/Server.cpp Server/Server.h $(objdir)Socket.o $(objdir)Database.o $(objdir)Operation.o $(objdir)Util.o
$(CC) $(CFLAGS) Server/Server.cpp $(inc) -o $(objdir)Server.o
$(objdir)SPARQLquery.o: Query/SPARQLquery.cpp Query/SPARQLquery.h $(objdir)BasicQuery.o
$(CC) $(CFLAGS) Query/SPARQLquery.cpp $(inc) -o $(objdir)SPARQLquery.o
$(objdir)GeneralEvaluation.o: Query/GeneralEvaluation.cpp Query/GeneralEvaluation.h $(objdir)QueryParser.o $(objdir)QueryTree.o \
$(objdir)SPARQLquery.o $(objdir)Varset.o $(objdir)KVstore.o $(objdir)ResultFilter.o $(objdir)Strategy.o $(objdir)StringIndex.o
$(CC) $(CFLAGS) Query/GeneralEvaluation.cpp $(inc) -o $(objdir)GeneralEvaluation.o
#$(objdir)SIStorage.o: KVstore/SITree/storage/SIStorage.cpp KVstore/SITree/storage/SIStorage.h Util/Util.h
#$(CC) $(CFLAGS) KVstore/SITree/storage/SIStorage.cpp -o $(objdir)SIStorage.o $(def64IO)
#$(objdir)ISStorage.o: KVstore/ISTree/storage/ISStorage.cpp KVstore/ISTree/storage/ISStorage.h Util/Util.h
#$(CC) $(CFLAGS) KVstore/ISTree/storage/ISStorage.cpp -o $(objdir)ISStorage.o $(def64IO)
#$(objdir)VSTree.o $(objdir)VSTree.o: VSTree/VSTree.cpp VSTree/VSTree.h $(objdir)EntryBuffer.o $(objdir)LRUCache.o $(objdir)VNode.o
#$(CC) $(CFLAGS) VSTree/VSTree.cpp $(inc) -o $(objdir)VSTree.o $(def64IO)
#$(objdir)EntryBuffer.o: VSTree/EntryBuffer.cpp VSTree/EntryBuffer.h Signature/SigEntry.h
#$(CC) $(CFLAGS) VSTree/EntryBuffer.cpp $(inc) -o $(objdir)EntryBuffer.o $(def64IO)
#$(objdir)LRUCache.o: VSTree/LRUCache.cpp VSTree/LRUCache.h VSTree/VNode.h
#$(CC) $(CFLAGS) VSTree/LRUCache.cpp $(inc) -o $(objdir)LRUCache.o $(def64IO)
#$(objdir)VNode.o: VSTree/VNode.cpp VSTree/VNode.h
#$(CC) $(CFLAGS) VSTree/VNode.cpp $(inc) -o $(objdir)VNode.o $(def64IO)
$(lib_antlr):
rm -rf tools/libantlr3c-3.4/
cd tools; tar -xzvf libantlr3c-3.4.tar.gz;
cd tools; cd libantlr3c-3.4/; ./configure -enable-64bit; make;
rm -rf lib/libantlr.a
ar -crv lib/libantlr.a tools/libantlr3c-3.4/*.o
#NOTICE: update the sparql.tar.gz if Sparql* in Parser are changed manually
rm -rf Parser/Sparql*
cd tools; tar -xzvf sparql.tar.gz; mv Sparql* ../Parser/;
$(api_cpp): $(objdir)Socket.o
$(MAKE) -C api/cpp/src
$(api_java):
$(MAKE) -C api/java/src
.PHONY: clean dist tarball api_example gtest sumlines
clean:
rm -rf lib/libantlr.a
$(MAKE) -C api/cpp/src clean
$(MAKE) -C api/cpp/example clean
$(MAKE) -C api/java/src clean
$(MAKE) -C api/java/example clean
#$(MAKE) -C KVstore clean
rm -rf $(exedir)g* $(objdir)*.o $(objdir)*.d $(exedir).gserver*
#rm -rf .project .cproject .settings just for eclipse
#rm -rf cscope* just for vim
find . -type f -print|grep -E "\.d"|xargs rm
dist: clean
rm -rf *.nt *.n3 .oebug/*.log .tmp/*.oat *.txt *.ob
rm -rf tools/libantlr3c-3.4 lib/libantlr.a Parser/Sparql*
#rm -rf Parser/SparqlLexer* Parser/SparlParser.cpp
rm -rf cscope* .cproject .settings tags
rm -rf *.info
tarball:
tar -czvf devGstore.tar.gz api bin lib tools .oebug .tmp .objs test docs data makefile \
Main Database KVstore Util Query Signature VSTree Parser Server README.md init.conf NOTES.md StringIndex
APIexample: $(api_cpp) $(api_java)
$(MAKE) -C api/cpp/example
$(MAKE) -C api/java/example
gtest: $(objdir)gtest.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gtest $(objdir)gtest.o $(objfile) lib/libantlr.a $(library)
$(objdir)gtest.o: test/gtest.cpp
$(CC) $(CFLAGS) test/gtest.cpp $(inc) -o $(objdir)gtest.o
$(exedir)gadd: $(objdir)gadd.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gadd $(objdir)gadd.o $(objfile) lib/libantlr.a $(library)
$(objdir)gadd.o: Main/gadd.cpp
$(CC) $(CFLAGS) Main/gadd.cpp $(inc) -o $(objdir)gadd.o
$(exedir)gsub: $(objdir)gsub.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gsub $(objdir)gsub.o $(objfile) lib/libantlr.a $(library)
$(objdir)gsub.o: Main/gsub.cpp
$(CC) $(CFLAGS) Main/gsub.cpp $(inc) -o $(objdir)gsub.o
sumlines:
bash test/sumline.sh
tag:
ctags -R
idx:
find `realpath .` -name "*.h" -o -name "*.c" -o -name "*.cpp" > cscope.files
cscope -bkq #-i cscope.files
cover:
bash test/cover.sh
fulltest:
#NOTICE:compile gstore with -O2 only
#setup new virtuoso and configure it
cp test/full_test.sh ~
cd ~
bash full_test.sh

114
test/makefile
View File

@ -1,114 +0,0 @@
#help for make
#http://www.cnblogs.com/wang_yb/p/3990952.html
#https://segmentfault.com/a/1190000000349917
#http://blog.csdn.net/cuiyifang/article/details/7910268
#compile parameters
CC = g++
CFLAGS = -c -g -Wall
# paths
VPATH = .:Util:Main:Parser:Query:KVstore:Database:Signature:VSTree:Server
SRC_PATH = $(foreach dir, $(subst:,,$(VPATH)), $(wildcard $(dir)/,*.cpp))
OBJ_DIR = .objs/
#OBJ_PATH = $(addprefix $(OBJ_DIR)/, $(subst:,,$(VPATH)))
#MAKE_OBJECT_DIR := $(shell mkdir -p $(OBJ_DIR $(OBJ_PATH)))
OBJS = $(addprefix $(OBJ_DIR),$(subst .cpp,.o,$(SRC_PATH)))
EXE_DIR = bin/
ANTLR = lib/libantlr.a
INC = -I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/INClude
#add -lreadline -ltermcap if using readline or objs contain readline
LIBRARY = -ltermcap -lreadline -L./lib -lantlr
all: $(ANTLR) $(EXE_DIR)gload $(EXE_DIR)gserver $(EXE_DIR)gclient $(EXE_DIR)gquery $(EXE_DIR)gconsole API
#executables begin
$(EXE_DIR)gload: $(OBJS) $(ANTLR)
$(CC) -g -o $@ $(OBJS) $(LIBRARY)
$(EXE_DIR)gquery: $(OBJS) $(ANTLR)
$(CC) -g -o $@ $(OBJS) $(LIBRARY)
$(EXE_DIR)gserver: $(OBJS) $(ANTLR)
$(CC) -g -o $@ $(OBJS) $(LIBRARY)
$(EXE_DIR)gclient: $(OBJS) $(ANTLR)
$(CC) -g -o $@ $(OBJS) $(LIBRARY)
$(EXE_DIR)gconsole: $(OBJS) $(ANTLR) API
$(CC) -g -o $@ $(OBJS) $(LIBRARY) -L./api/cpp/lib -lgstoreconnector
#executables end
$(OBJ_DIR)%.o: %.cpp %.cpp.d
$(CC) -o $@ $< $(INC) $(CFLAGS)
DEPS = $(OBJS:.o=cpp.d)
$(DEPS): $(OBJ_DIR)%.cpp.d: %.cpp
$(CC) $< -MM $(INC) > $(OBJ_DIR)$<.d
$(ANTLR):
rm -rf tools/libantlr3c-3.4/
cd tools; tar -xzvf libantlr3c-3.4.tar.gz;
cd tools; cd libantlr3c-3.4/; ./configure -enable-64bit; make;
rm -rf lib/libantlr.a
ar -crv lib/libantlr.a tools/libantlr3c-3.4/*.o
#NOTICE: update the sparql.tar.gz if Sparql* in Parser are changed manually
rm -rf Parser/Sparql*
cd tools; tar -xzvf sparql.tar.gz; mv Sparql* ../Parser/;
API:
$(MAKE) -C api/cpp/src
#$(MAKE) -C api/cpp/example
$(MAKE) -C api/java/src
#$(MAKE) -C api/java/example
.PHONY: clean dist tarball api_example gtest sumlines
clean:
$(MAKE) -C api/cpp/src clean
$(MAKE) -C api/cpp/example clean
$(MAKE) -C api/java/src clean
$(MAKE) -C api/java/example clean
#$(MAKE) -C KVstore clean
rm -rf $(EXE_DIR)g* $(OBJ_DIR)*.o
#rm -rf .project .cproject .settings just for eclipse
#rm -rf cscope* just for vim
dist: clean
rm -rf *.nt *.n3 .debug/*.log .tmp/*.dat *.txt *.db
rm -rf tools/libantlr3c-3.4 lib/libantlr.a Parser/Sparql*
tarball:
tar -czvf devGstore.tar.gz api bin lib tools .debug .tmp .objs test docs data makefile \
Main Database KVstore Util Query Signature VSTree Parser Server LICENSE README.md
api_example: API
$(MAKE) -C api/cpp/example
$(MAKE) -C api/java/example
gtest: $(OBJ_DIR)gtest.o $(OBJS)
$(CC) -g -o $(EXE_DIR)gtest $(OBJ_DIR)gtest.o $(OBJS) lib/libantlr.a $(LIBRARY)
$(OBJ_DIR)gtest.o: test/gtest.cpp
$(CC) $(CFLAGS) test/gtest.cpp $(INC) -o $(OBJ_DIR)gtest.o
sumlines:
bash test/sumline.sh

446
test/raw_makefile Normal file
View File

@ -0,0 +1,446 @@
#help for make
#http://www.cnblogs.com/wang_yb/p/3990952.html
#https://segmentfault.com/a/1190000000349917
#http://blog.csdn.net/cuiyifang/article/details/7910268
#to use gprof to analyse efficience of the program:
#http://blog.chinaunix.net/uid-25194149-id-3215487.html
#to use gcov and lcov
#Notice that optimization should not be used here
#http://blog.163.com/bobile45@126/blog/static/96061992201382025729313/
#gcov -a main.cpp
#lcov --directory . --capture --output-file dig.info
#genhtml --output-directory . --frames --show-details dig.info
#to use doxygen+graphviz+htmlhelp to generate document from source code:
#http://www.doxygen.nl/
#(also include good comments norm)
#http://blog.csdn.net/u010740725/article/details/51387810
#NOTICE: to speed up the make process, use make -j4
#use -j8 or higher may cause error
#http://blog.csdn.net/cscrazybing/article/details/50789482
#http://blog.163.com/liuhonggaono1@126/blog/static/10497901201210254622141/
#TODO:the dependences are not complete!
#compile parameters
CC = ccache g++
#CC = g++
#the optimazition level of gcc/g++
#http://blog.csdn.net/hit_090420216/article/details/44900215
#NOTICE: -O2 is recommended, while -O3 is dangerous
#when developing, not use -O because it will disturb the normal
#routine. use it for test and release.
#CFLAGS = -c -Wall -g #-fprofile-arcs -ftest-coverage #-pg
#EXEFLAG = -g #-fprofile-arcs -ftest-coverage #-pg
#-coverage
CFLAGS = -c -Wall -O2
EXEFLAG = -O2
#add -lreadline -ltermcap if using readline or objs contain readline
library = -ltermcap -lreadline -L./lib -lantlr -lgcov
def64IO = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE
# paths
objdir = .objs/
exedir = bin/
lib_antlr = lib/libantlr.a
api_cpp = api/cpp/lib/libgstoreconnector.a
api_java = api/java/lib/GstoreJavaAPI.jar
# objects
#sstreeobj = $(objdir)Tree.o $(objdir)Storage.o $(objdir)Node.o $(objdir)IntlNode.o $(objdir)LeafNode.o $(objdir)Heap.o
sitreeobj = $(objdir)SITree.o $(objdir)SIStorage.o $(objdir)SINode.o $(objdir)SIIntlNode.o $(objdir)SILeafNode.o $(objdir)SIHeap.o
istreeobj = $(objdir)ISTree.o $(objdir)ISStorage.o $(objdir)ISNode.o $(objdir)ISIntlNode.o $(objdir)ISLeafNode.o $(objdir)ISHeap.o
kvstoreobj = $(objdir)KVstore.o $(sitreeobj) $(istreeobj) #$(sstreeobj)
utilobj = $(objdir)Util.o $(objdir)Bstr.o $(objdir)Stream.o $(objdir)Triple.o $(objdir)BloomFilter.o
queryobj = $(objdir)SPARQLquery.o $(objdir)BasicQuery.o $(objdir)ResultSet.o $(objdir)IDList.o \
$(objdir)Varset.o $(objdir)QueryTree.o $(objdir)ResultFilter.o $(objdir)GeneralEvaluation.o
signatureobj = $(objdir)SigEntry.o $(objdir)Signature.o
vstreeobj = $(objdir)VSTree.o $(objdir)EntryBuffer.o $(objdir)LRUCache.o $(objdir)VNode.o
stringindexobj = $(objdir)StringIndex.o
parserobj = $(objdir)RDFParser.o $(objdir)SparqlParser.o $(objdir)DBparser.o \
$(objdir)SparqlLexer.o $(objdir)TurtleParser.o $(objdir)QueryParser.o
serverobj = $(objdir)Operation.o $(objdir)Server.o $(objdir)Client.o $(objdir)Socket.o
databaseobj = $(objdir)Database.o $(objdir)Join.o $(objdir)Strategy.o
objfile = $(kvstoreobj) $(vstreeobj) $(stringindexobj) $(parserobj) $(serverobj) $(databaseobj) \
$(utilobj) $(signatureobj) $(queryobj)
inc = -I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include
#auto generate dependencies
# http://blog.csdn.net/gmpy_tiger/article/details/51849474
# http://blog.csdn.net/jeffrey0000/article/details/12421317
#gtest
TARGET = $(exedir)gbuild $(exedir)gserver $(exedir)gclient $(exedir)gquery $(exedir)gconsole $(api_java) $(exedir)gadd $(exedir)gsub
all: $(TARGET)
test_index: test_index.cpp
$(CC) $(EXEFLAG) -o test_index test_index.cpp $(objfile) $(library)
#BETTER: use for loop to reduce the lines
#NOTICE: g++ -MM will run error if linking failed, like Database.h/../SparlParser.h/../antlr3.h
#executables begin
#NOTICE:not include g*.o in objfile due to multiple definitions of main()
$(exedir)gbuild: $(lib_antlr) $(objdir)gbuild.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gbuild $(objdir)gbuild.o $(objfile) $(library)
$(exedir)gquery: $(lib_antlr) $(objdir)gquery.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gquery $(objdir)gquery.o $(objfile) $(library)
$(exedir)gserver: $(lib_antlr) $(objdir)gserver.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gserver $(objdir)gserver.o $(objfile) $(library)
$(exedir)gclient: $(lib_antlr) $(objdir)gclient.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gclient $(objdir)gclient.o $(objfile) $(library)
$(exedir)gconsole: $(lib_antlr) $(objdir)gconsole.o $(objfile) $(api_cpp)
$(CC) $(EXEFLAG) -o $(exedir)gconsole $(objdir)gconsole.o $(objfile) $(library) -L./api/cpp/lib -lgstoreconnector
#executables end
#objects in Main/ begin
$(objdir)gbuild.o: Main/gbuild.cpp Database/Database.h Util/Util.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gbuild.cpp $(inc) -o $(objdir)gbuild.o
$(objdir)gquery.o: Main/gquery.cpp Database/Database.h Util/Util.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gquery.cpp $(inc) -o $(objdir)gquery.o #-DREADLINE_ON
#add -DREADLINE_ON if using readline
$(objdir)gserver.o: Main/gserver.cpp Server/Server.h Util/Util.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gserver.cpp $(inc) -o $(objdir)gserver.o
$(objdir)gclient.o: Main/gclient.cpp Server/Client.h Util/Util.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gclient.cpp $(inc) -o $(objdir)gclient.o #-DREADLINE_ON
$(objdir)gconsole.o: Main/gconsole.cpp Database/Database.h Util/Util.h api/cpp/src/GstoreConnector.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gconsole.cpp $(inc) -o $(objdir)gconsole.o -I./api/cpp/src/ #-DREADLINE_ON
#objects in Main/ end
#objects in kvstore/ begin
#objects in sstree/ begin
#$(objdir)Tree.o: KVstore/SSTree/Tree.cpp KVstore/SSTree/Tree.h $(objdir)Stream.o
#$(CC) $(CFLAGS) KVstore/SSTree/Tree.cpp -o $(objdir)Tree.o
#$(objdir)Storage.o: KVstore/SSTree/storage/Storage.cpp KVstore/SSTree/storage/Storage.h $(objdir)Util.o
#$(CC) $(CFLAGS) KVstore/SSTree/storage/Storage.cpp -o $(objdir)Storage.o $(def64IO)
#$(objdir)Node.o: KVstore/SSTree/node/Node.cpp KVstore/SSTree/node/Node.h $(objdir)Util.o
#$(CC) $(CFLAGS) KVstore/SSTree/node/Node.cpp -o $(objdir)Node.o
#$(objdir)IntlNode.o: KVstore/SSTree/node/IntlNode.cpp KVstore/SSTree/node/IntlNode.h
#$(CC) $(CFLAGS) KVstore/SSTree/node/IntlNode.cpp -o $(objdir)IntlNode.o
#$(objdir)LeafNode.o: KVstore/SSTree/node/LeafNode.cpp KVstore/SSTree/node/LeafNode.h
#$(CC) $(CFLAGS) KVstore/SSTree/node/LeafNode.cpp -o $(objdir)LeafNode.o
#$(objdir)Heap.o: KVstore/SSTree/heap/Heap.cpp KVstore/SSTree/heap/Heap.h $(objdir)Util.o
#$(CC) $(CFLAGS) KVstore/SSTree/heap/Heap.cpp -o $(objdir)Heap.o
#objects in sstree/ end
#objects in sitree/ begin
$(objdir)SITree.o: KVstore/SITree/SITree.cpp KVstore/SITree/SITree.h $(objdir)Stream.o
$(CC) $(CFLAGS) KVstore/SITree/SITree.cpp -o $(objdir)SITree.o
$(objdir)SIStorage.o: KVstore/SITree/storage/SIStorage.cpp KVstore/SITree/storage/SIStorage.h $(objdir)Util.o
$(CC) $(CFLAGS) KVstore/SITree/storage/SIStorage.cpp -o $(objdir)SIStorage.o $(def64IO)
$(objdir)SINode.o: KVstore/SITree/node/SINode.cpp KVstore/SITree/node/SINode.h $(objdir)Util.o
$(CC) $(CFLAGS) KVstore/SITree/node/SINode.cpp -o $(objdir)SINode.o
$(objdir)SIIntlNode.o: KVstore/SITree/node/SIIntlNode.cpp KVstore/SITree/node/SIIntlNode.h
$(CC) $(CFLAGS) KVstore/SITree/node/SIIntlNode.cpp -o $(objdir)SIIntlNode.o
$(objdir)SILeafNode.o: KVstore/SITree/node/SILeafNode.cpp KVstore/SITree/node/SILeafNode.h
$(CC) $(CFLAGS) KVstore/SITree/node/SILeafNode.cpp -o $(objdir)SILeafNode.o
$(objdir)SIHeap.o: KVstore/SITree/heap/SIHeap.cpp KVstore/SITree/heap/SIHeap.h $(objdir)Util.o
$(CC) $(CFLAGS) KVstore/SITree/heap/SIHeap.cpp -o $(objdir)SIHeap.o
#objects in sitree/ end
#objects in istree/ begin
$(objdir)ISTree.o: KVstore/ISTree/ISTree.cpp KVstore/ISTree/ISTree.h $(objdir)Stream.o
$(CC) $(CFLAGS) KVstore/ISTree/ISTree.cpp -o $(objdir)ISTree.o
$(objdir)ISStorage.o: KVstore/ISTree/storage/ISStorage.cpp KVstore/ISTree/storage/ISStorage.h $(objdir)Util.o
$(CC) $(CFLAGS) KVstore/ISTree/storage/ISStorage.cpp -o $(objdir)ISStorage.o $(def64IO)
$(objdir)ISNode.o: KVstore/ISTree/node/ISNode.cpp KVstore/ISTree/node/ISNode.h $(objdir)Util.o
$(CC) $(CFLAGS) KVstore/ISTree/node/ISNode.cpp -o $(objdir)ISNode.o
$(objdir)ISIntlNode.o: KVstore/ISTree/node/ISIntlNode.cpp KVstore/ISTree/node/ISIntlNode.h
$(CC) $(CFLAGS) KVstore/ISTree/node/ISIntlNode.cpp -o $(objdir)ISIntlNode.o
$(objdir)ISLeafNode.o: KVstore/ISTree/node/ISLeafNode.cpp KVstore/ISTree/node/ISLeafNode.h
$(CC) $(CFLAGS) KVstore/ISTree/node/ISLeafNode.cpp -o $(objdir)ISLeafNode.o
$(objdir)ISHeap.o: KVstore/ISTree/heap/ISHeap.cpp KVstore/ISTree/heap/ISHeap.h $(objdir)Util.o
$(CC) $(CFLAGS) KVstore/ISTree/heap/ISHeap.cpp -o $(objdir)ISHeap.o
#objects in istree/ end
$(objdir)KVstore.o: KVstore/KVstore.cpp KVstore/KVstore.h KVstore/Tree.h
$(CC) $(CFLAGS) KVstore/KVstore.cpp $(inc) -o $(objdir)KVstore.o
#objects in kvstore/ end
#objects in Database/ begin
$(objdir)Database.o: Database/Database.cpp Database/Database.h \
$(objdir)IDList.o $(objdir)ResultSet.o $(objdir)SPARQLquery.o \
$(objdir)BasicQuery.o $(objdir)Triple.o $(objdir)SigEntry.o \
$(objdir)KVstore.o $(objdir)VSTree.o $(objdir)DBparser.o \
$(objdir)Util.o $(objdir)RDFParser.o $(objdir)Join.o $(objdir)GeneralEvaluation.o $(objdir)StringIndex.o
$(CC) $(CFLAGS) Database/Database.cpp $(inc) -o $(objdir)Database.o
$(objdir)Join.o: Database/Join.cpp Database/Join.h $(objdir)IDList.o $(objdir)BasicQuery.o $(objdir)Util.o\
$(objdir)KVstore.o $(objdir)Util.o $(objdir)SPARQLquery.o
$(CC) $(CFLAGS) Database/Join.cpp $(inc) -o $(objdir)Join.o
$(objdir)Strategy.o: Database/Strategy.cpp Database/Strategy.h $(objdir)SPARQLquery.o $(objdir)BasicQuery.o \
$(objdir)Triple.o $(objdir)IDList.o $(objdir)KVstore.o $(objdir)VSTree.o $(objdir)Util.o $(objdir)Join.o $(objdir)ResultFilter.o
$(CC) $(CFLAGS) Database/Strategy.cpp $(inc) -o $(objdir)Strategy.o
#objects in Database/ end
#objects in Query/ begin
$(objdir)IDList.o: Query/IDList.cpp Query/IDList.h
$(CC) $(CFLAGS) Query/IDList.cpp $(inc) -o $(objdir)IDList.o
$(objdir)SPARQLquery.o: Query/SPARQLquery.cpp Query/SPARQLquery.h $(objdir)BasicQuery.o
$(CC) $(CFLAGS) Query/SPARQLquery.cpp $(inc) -o $(objdir)SPARQLquery.o
$(objdir)BasicQuery.o: Query/BasicQuery.cpp Query/BasicQuery.h $(objdir)Signature.o
$(CC) $(CFLAGS) Query/BasicQuery.cpp $(inc) -o $(objdir)BasicQuery.o
$(objdir)ResultSet.o: Query/ResultSet.cpp Query/ResultSet.h $(objdir)Stream.o
$(CC) $(CFLAGS) Query/ResultSet.cpp $(inc) -o $(objdir)ResultSet.o
$(objdir)Varset.o: Query/Varset.cpp Query/Varset.h
$(CC) $(CFLAGS) Query/Varset.cpp $(inc) -o $(objdir)Varset.o
$(objdir)QueryTree.o: Query/QueryTree.cpp Query/QueryTree.h $(objdir)Varset.o
$(CC) $(CFLAGS) Query/QueryTree.cpp $(inc) -o $(objdir)QueryTree.o
$(objdir)ResultFilter.o: Query/ResultFilter.cpp Query/ResultFilter.h $(objdir)BasicQuery.o $(objdir)SPARQLquery.o $(objdir)Util.o
$(CC) $(CFLAGS) Query/ResultFilter.cpp $(inc) -o $(objdir)ResultFilter.o
#no more using $(objdir)Database.o
$(objdir)GeneralEvaluation.o: Query/GeneralEvaluation.cpp Query/GeneralEvaluation.h $(objdir)QueryParser.o $(objdir)QueryTree.o \
$(objdir)SPARQLquery.o $(objdir)Varset.o $(objdir)KVstore.o $(objdir)ResultFilter.o $(objdir)Strategy.o $(objdir)StringIndex.o
$(CC) $(CFLAGS) Query/GeneralEvaluation.cpp $(inc) -o $(objdir)GeneralEvaluation.o
#objects in Query/ end
#objects in Signature/ begin
$(objdir)SigEntry.o: Signature/SigEntry.cpp Signature/SigEntry.h $(objdir)Signature.o
$(CC) $(CFLAGS) Signature/SigEntry.cpp $(inc) -o $(objdir)SigEntry.o
$(objdir)Signature.o: Signature/Signature.cpp Signature/Signature.h
$(CC) $(CFLAGS) Signature/Signature.cpp $(inc) -o $(objdir)Signature.o
#objects in Signature/ end
#objects in Util/ begin
$(objdir)Util.o: Util/Util.cpp Util/Util.h
$(CC) $(CFLAGS) Util/Util.cpp -o $(objdir)Util.o
$(objdir)Stream.o: Util/Stream.cpp Util/Stream.h $(objdir)Util.o $(objdir)Bstr.o
$(CC) $(CFLAGS) Util/Stream.cpp -o $(objdir)Stream.o $(def64IO)
$(objdir)Bstr.o: Util/Bstr.cpp Util/Bstr.h $(objdir)Util.o
$(CC) $(CFLAGS) Util/Bstr.cpp -o $(objdir)Bstr.o
$(objdir)Triple.o: Util/Triple.cpp Util/Triple.h $(objdir)Util.o
$(CC) $(CFLAGS) Util/Triple.cpp -o $(objdir)Triple.o
$(objdir)BloomFilter.o: Util/BloomFilter.cpp Util/BloomFilter.h $(objdir)Util.o
$(CC) $(CFLAGS) Util/BloomFilter.cpp -o $(objdir)BloomFilter.o
#objects in util/ end
#objects in VSTree/ begin
$(objdir)VSTree.o: VSTree/VSTree.cpp VSTree/VSTree.h $(objdir)EntryBuffer.o $(objdir)LRUCache.o $(objdir)VNode.o
$(CC) $(CFLAGS) VSTree/VSTree.cpp $(inc) -o $(objdir)VSTree.o $(def64IO)
$(objdir)EntryBuffer.o: VSTree/EntryBuffer.cpp VSTree/EntryBuffer.h Signature/SigEntry.h
$(CC) $(CFLAGS) VSTree/EntryBuffer.cpp $(inc) -o $(objdir)EntryBuffer.o $(def64IO)
$(objdir)LRUCache.o: VSTree/LRUCache.cpp VSTree/LRUCache.h VSTree/VNode.h
$(CC) $(CFLAGS) VSTree/LRUCache.cpp $(inc) -o $(objdir)LRUCache.o $(def64IO)
$(objdir)VNode.o: VSTree/VNode.cpp VSTree/VNode.h
$(CC) $(CFLAGS) VSTree/VNode.cpp $(inc) -o $(objdir)VNode.o $(def64IO)
#objects in VSTree/ end
#objects in StringIndex/ begin
$(objdir)StringIndex.o: StringIndex/StringIndex.cpp StringIndex/StringIndex.h $(objdir)KVstore.o $(objdir)Util.o
$(CC) $(CFLAGS) StringIndex/StringIndex.cpp $(inc) -o $(objdir)StringIndex.o
#objects in StringIndex/ end
#objects in Parser/ begin
$(objdir)DBparser.o: Parser/DBparser.cpp Parser/DBparser.h $(objdir)SparqlParser.o $(objdir)SparqlLexer.o $(objdir)Triple.o
$(CC) $(CFLAGS) Parser/DBparser.cpp $(inc) -o $(objdir)DBparser.o
$(objdir)SparqlParser.o: Parser/SparqlParser.c Parser/SparqlParser.h
gcc $(CFLAGS) Parser/SparqlParser.c $(inc) -o $(objdir)SparqlParser.o
$(objdir)SparqlLexer.o: Parser/SparqlLexer.c Parser/SparqlLexer.h
gcc $(CFLAGS) Parser/SparqlLexer.c $(inc) -o $(objdir)SparqlLexer.o
$(objdir)TurtleParser.o: Parser/TurtleParser.cpp Parser/TurtleParser.h Parser/Type.h
gcc $(CFLAGS) Parser/TurtleParser.cpp $(inc) -o $(objdir)TurtleParser.o
$(objdir)RDFParser.o: Parser/RDFParser.cpp Parser/RDFParser.h $(objdir)TurtleParser.o $(objdir)Triple.o
gcc $(CFLAGS) Parser/RDFParser.cpp $(inc) -o $(objdir)RDFParser.o
$(objdir)QueryParser.o: Parser/QueryParser.cpp Parser/QueryParser.h $(objdir)SparqlParser.o $(objdir)SparqlLexer.o $(objdir)QueryTree.o
$(CC) $(CFLAGS) Parser/QueryParser.cpp $(inc) -o $(objdir)QueryParser.o
#objects in Parser/ end
#objects in Server/ begin
$(objdir)Operation.o: Server/Operation.cpp Server/Operation.h
$(CC) $(CFLAGS) Server/Operation.cpp $(inc) -o $(objdir)Operation.o
$(objdir)Socket.o: Server/Socket.cpp Server/Socket.h
$(CC) $(CFLAGS) Server/Socket.cpp $(inc) -o $(objdir)Socket.o
$(objdir)Server.o: Server/Server.cpp Server/Server.h $(objdir)Socket.o $(objdir)Database.o $(objdir)Operation.o
$(CC) $(CFLAGS) Server/Server.cpp $(inc) -o $(objdir)Server.o
$(objdir)Client.o: Server/Client.cpp Server/Client.h $(objdir)Socket.o $(objdir)Util.o
$(CC) $(CFLAGS) Server/Client.cpp $(inc) -o $(objdir)Client.o
#objects in Server/ end
$(lib_antlr):
rm -rf tools/libantlr3c-3.4/
cd tools; tar -xzvf libantlr3c-3.4.tar.gz;
cd tools; cd libantlr3c-3.4/; ./configure -enable-64bit; make;
rm -rf lib/libantlr.a
ar -crv lib/libantlr.a tools/libantlr3c-3.4/*.o
#NOTICE: update the sparql.tar.gz if Sparql* in Parser are changed manually
rm -rf Parser/Sparql*
cd tools; tar -xzvf sparql.tar.gz; mv Sparql* ../Parser/;
$(api_cpp): $(objdir)Socket.o
$(MAKE) -C api/cpp/src
$(api_java):
$(MAKE) -C api/java/src
.PHONY: clean dist tarball api_example gtest sumlines
clean:
rm -rf lib/libantlr.a
$(MAKE) -C api/cpp/src clean
$(MAKE) -C api/cpp/example clean
$(MAKE) -C api/java/src clean
$(MAKE) -C api/java/example clean
#$(MAKE) -C KVstore clean
rm -rf $(exedir)g* $(objdir)*.o $(exedir).gserver*
#rm -rf .project .cproject .settings just for eclipse
#rm -rf cscope* just for vim
dist: clean
rm -rf *.nt *.n3 .debug/*.log .tmp/*.dat *.txt *.db
rm -rf tools/libantlr3c-3.4 lib/libantlr.a Parser/Sparql*
#rm -rf Parser/SparqlLexer* Parser/SparlParser.cpp
rm -rf cscope* .cproject .settings tags
rm -rf *.info
tarball:
tar -czvf devGstore.tar.gz api bin lib tools .debug .tmp .objs test docs data makefile \
Main Database KVstore Util Query Signature VSTree Parser Server README.md init.conf NOTES.md StringIndex
APIexample: $(api_cpp) $(api_java)
$(MAKE) -C api/cpp/example
$(MAKE) -C api/java/example
gtest: $(objdir)gtest.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gtest $(objdir)gtest.o $(objfile) lib/libantlr.a $(library)
$(objdir)gtest.o: test/gtest.cpp
$(CC) $(CFLAGS) test/gtest.cpp $(inc) -o $(objdir)gtest.o
$(exedir)gadd: $(objdir)gadd.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gadd $(objdir)gadd.o $(objfile) lib/libantlr.a $(library)
$(objdir)gadd.o: Main/gadd.cpp
$(CC) $(CFLAGS) Main/gadd.cpp $(inc) -o $(objdir)gadd.o
$(exedir)gsub: $(objdir)gsub.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gsub $(objdir)gsub.o $(objfile) lib/libantlr.a $(library)
$(objdir)gsub.o: Main/gsub.cpp
$(CC) $(CFLAGS) Main/gsub.cpp $(inc) -o $(objdir)gsub.o
sumlines:
bash test/sumline.sh
tag:
ctags -R
idx:
find `realpath .` -name "*.h" -o -name "*.c" -o -name "*.cpp" > cscope.files
cscope -bkq #-i cscope.files
cover:
bash test/cover.sh
fulltest:
#NOTICE:compile gstore with -O2 only
#setup new virtuoso and configure it
cp test/full_test.sh ~
cd ~
bash full_test.sh