opensci
/
gStore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactor: release 0.5.0 

merge code in bookug branch

by zengli
This commit is contained in:
bookug 2017-08-07 13:39:34 +08:00
parent 50972de85d b6e4f36fc6
commit bf010ba035
235 changed files with 29826 additions and 8361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.gitignore vendored
View File

@ -91,9 +91,13 @@ tags
*.out
*.bak~
# queries
*.sql
*.sh
# modules
node_modules
# scripts
*.sh
# query results
*.info

0
api/cpp/lib/.gitignore → .tmp/web/.gitignore vendored
View File

2284
Database/Database.cpp
View File

File diff suppressed because it is too large Load Diff

154
Database/Database.h
View File

@ -28,13 +28,14 @@
class Database
{
public:
static const bool only_sub2idpre2id = true;
static const int internal = 100 * 1000;
void test();
void test_build_sig();
void test_join();
void printIDlist(int _i, int* _list, int _len, std::string _log);
void printPairList(int _i, int* _list, int _len, std::string _log);
//static const bool only_sub2idpre2id = true;
//static const int internal = 100 * 1000;
//void test();
//void test_build_sig();
//void test_join();
//void printIDlist(int _i, int* _list, int _len, std::string _log);
//void printPairList(int _i, int* _list, int _len, std::string _log);
//when encode EntitySig, one way uses STRING-hash, the other one uses ID-hash
//depending on this->encode_mode
@ -45,8 +46,10 @@ public:
void release(FILE* fp0);
~Database();
bool save();
bool load();
bool unload();
void clear();
int query(const string _query, ResultSet& _result_set, FILE* _fp = stdout);
//1. if subject of _triple doesn't exist,
@ -56,29 +59,43 @@ public:
bool build(const string& _rdf_file);
//interfaces to insert/delete from given rdf file
bool insert(std::string _rdf_file);
bool remove(std::string _rdf_file);
bool insert(std::string _rdf_file, bool _is_restore = false);
bool remove(std::string _rdf_file, bool _is_restore = false);
/* name of this DB*/
bool backup();
bool restore();
//name of this DB
string getName();
/* root Path of this DB + sixTuplesFile */
//get infos
TYPE_TRIPLE_NUM getTripleNum();
TYPE_ENTITY_LITERAL_ID getEntityNum();
TYPE_ENTITY_LITERAL_ID getLiteralNum();
TYPE_ENTITY_LITERAL_ID getSubNum();
TYPE_PREDICATE_ID getPreNum();
//root Path of this DB + sixTuplesFile
string getSixTuplesFile();
/* root Path of this DB + signatureBFile */
//root Path of this DB + signatureBFile
string getSignatureBFile();
/* root Path of this DB + DBInfoFile */
//root Path of this DB + DBInfoFile
string getDBInfoFile();
//id tuples file
string getIDTuplesFile();
private:
string name;
string store_path;
bool is_active;
int triples_num;
int entity_num;
int sub_num;
int pre_num;
int literal_num;
TYPE_TRIPLE_NUM triples_num;
TYPE_ENTITY_LITERAL_ID entity_num;
TYPE_ENTITY_LITERAL_ID sub_num;
//BETTER: add object num
TYPE_PREDICATE_ID pre_num;
TYPE_ENTITY_LITERAL_ID literal_num;
int encode_mode;
@ -94,56 +111,97 @@ private:
//six tuples: <sub pre obj sid pid oid>
string six_tuples_file;
//B means binary
string signature_binary_file;
//id tuples file
string id_tuples_file;
string update_log;
string update_log_since_backup;
//pre2num mapping
TNUM* pre2num;
TYPE_TRIPLE_NUM* pre2num;
//valid: check from minNumPID to maxNumPID
int maxNumPID, minNumPID;
TYPE_PREDICATE_ID maxNumPID, minNumPID;
void setPreMap();
//TODO: set the buffer capacity as dynamic according to the current memory usage
//string buffer
Buffer* entity_buffer;
//unsigned offset; //maybe let id start from an offset
unsigned entity_buffer_size;
Buffer* literal_buffer;
unsigned literal_buffer_size;
void setStringBuffer();
void warmUp();
//BETTER+TODO:add a predicate buffer for ?p query
//BETTER:add a predicate buffer for ?p query
//However, I think this is not necessary because ?p is rare and the p2xx tree is small enough
void check();
//used for multiple threads
void load_vstree(unsigned _vstree_size);
void load_entity2id(int _mode);
void load_id2entity(int _mode);
void load_literal2id(int _mode);
void load_id2literal(int _mode);
void load_predicate2id(int _mode);
void load_id2predicate(int _mode);
void load_sub2values(int _mode);
void load_obj2values(int _mode);
void load_pre2values(int _mode);
// functions used to build cache
void load_cache();
void get_important_preID();
std::vector <TYPE_PREDICATE_ID> important_preID;
void load_important_sub2values();
void load_important_obj2values();
void load_candidate_pre2values();
void build_CacheOfPre2values();
void build_CacheOfSub2values();
void build_CacheOfObj2values();
void get_important_subID();
void get_important_objID();
void get_candidate_preID();
std::priority_queue <KEY_SIZE_VALUE> candidate_preID;
std::priority_queue <KEY_SIZE_VALUE> important_subID;
std::priority_queue <KEY_SIZE_VALUE> important_objID;
//triple num per group for insert/delete
//can not be too high, otherwise the heap will over
static const int GROUP_SIZE = 1000;
//manage the ID allocate and garbage
static const int START_ID_NUM = 0;
static const TYPE_ENTITY_LITERAL_ID START_ID_NUM = 0;
//static const int START_ID_NUM = 1000;
/////////////////////////////////////////////////////////////////////////////////
//NOTICE:error if >= LITERAL_FIRST_ID
string free_id_file_entity; //the first is limitID, then free id list
int limitID_entity; //the current maxium ID num(maybe not used so much)
TYPE_ENTITY_LITERAL_ID limitID_entity; //the current maxium ID num(maybe not used so much)
BlockInfo* freelist_entity; //free id list, reuse BlockInfo for Storage class
int allocEntityID();
void freeEntityID(int _id);
TYPE_ENTITY_LITERAL_ID allocEntityID();
void freeEntityID(TYPE_ENTITY_LITERAL_ID _id);
/////////////////////////////////////////////////////////////////////////////////
//NOTICE:error if >= 2*LITERAL_FIRST_ID
string free_id_file_literal;
int limitID_literal;
TYPE_ENTITY_LITERAL_ID limitID_literal;
BlockInfo* freelist_literal;
int allocLiteralID();
void freeLiteralID(int _id);
TYPE_ENTITY_LITERAL_ID allocLiteralID();
void freeLiteralID(TYPE_ENTITY_LITERAL_ID _id);
/////////////////////////////////////////////////////////////////////////////////
//NOTICE:error if >= 2*LITERAL_FIRST_ID
string free_id_file_predicate;
int limitID_predicate;
TYPE_PREDICATE_ID limitID_predicate;
BlockInfo* freelist_predicate;
int allocPredicateID();
void freePredicateID(int _id);
TYPE_PREDICATE_ID allocPredicateID();
void freePredicateID(TYPE_PREDICATE_ID _id);
/////////////////////////////////////////////////////////////////////////////////
void initIDinfo(); //initialize the members
void resetIDinfo(); //reset the id info for build
void readIDinfo(); //read and build the free list
void writeIDinfo(); //write and empty the free list
void saveIDinfo(); //write and empty the free list
bool saveDBInfoFile();
bool loadDBInfoFile();
@ -155,14 +213,18 @@ private:
//encode Triple into Subject EntityBitSet
bool encodeTriple2SubEntityBitSet(EntityBitSet& _bitset, const Triple* _p_triple);
//NOTICE: the encodeTriple with Triple* is invalid now(not enocde the linkage of neighbor-predicate)
bool encodeTriple2SubEntityBitSet(EntityBitSet& _bitset, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
//encode Triple into Object EntityBitSet
bool encodeTriple2ObjEntityBitSet(EntityBitSet& _bitset, const Triple* _p_triple);
bool encodeTriple2ObjEntityBitSet(EntityBitSet& _bitset, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _sub_id);
bool calculateEntityBitSet(int _entity_id, EntityBitSet & _bitset);
bool calculateEntityBitSet(TYPE_ENTITY_LITERAL_ID _entity_id, EntityBitSet & _bitset);
//check whether the relative 3-tuples exist
//usually, through sp2olist
bool exist_triple(int _sub_id, int _pre_id, int _obj_id);
bool exist_triple(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
bool exist_triple(const TripleWithObjType& _triple);
//* _rdf_file denotes the path of the RDF file, where stores the rdf data
//* there are many step in this function, each one responds to an sub-function
@ -174,24 +236,24 @@ private:
//* 4. build: objID2subIDlist, <objIDpreID>2subIDlist objID2<preIDsubID>list
//encodeRDF_new invoke new rdfParser to solve task 1 & 2 in one time scan.
bool encodeRDF_new(const string _rdf_file);
void build_s2xx(int**);
void build_o2xx(int**);
void build_p2xx(int**);
void readIDTuples(ID_TUPLE*& _p_id_tuples);
void build_s2xx(ID_TUPLE*);
void build_o2xx(ID_TUPLE*);
void build_p2xx(ID_TUPLE*);
//insert and delete, notice that modify is not needed here
//we can read from file or use sparql syntax
bool insertTriple(const TripleWithObjType& _triple, vector<int>* _vertices = NULL, vector<int>* _predicates = NULL);
bool removeTriple(const TripleWithObjType& _triple, vector<int>* _vertices = NULL, vector<int>* _predicates = NULL);
bool insertTriple(const TripleWithObjType& _triple, vector<unsigned>* _vertices = NULL, vector<unsigned>* _predicates = NULL);
bool removeTriple(const TripleWithObjType& _triple, vector<unsigned>* _vertices = NULL, vector<unsigned>* _predicates = NULL);
//NOTICE:one by one is too costly, sort and insert/delete at a time will be better
int insert(const TripleWithObjType* _triples, int _triple_num);
unsigned insert(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, bool _is_restore=false);
//bool insert(const vector<TripleWithObjType>& _triples, vector<int>& _vertices, vector<int>& _predicates);
int remove(const TripleWithObjType* _triples, int _triple_num);
//bool remove(const vector<TripleWithObjType>& _triples, vector<int>& _vertices, vector<int>& _predicates);
unsigned remove(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, bool _is_restore=false);
bool sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file, int**& _p_id_tuples, int & _id_tuples_max);
bool literal2id_RDFintoSignature(const string _rdf_file, int** _p_id_tuples, int _id_tuples_max);
bool sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file);
//bool literal2id_RDFintoSignature(const string _rdf_file, int** _p_id_tuples, TYPE_TRIPLE_NUM _id_tuples_max);
bool objIDIsEntityID(int _id);
bool objIDIsEntityID(TYPE_ENTITY_LITERAL_ID _id);
//* join on the vector of CandidateList, available after retrieve from the VSTREE
//* and store the resut in _result_set
@ -202,6 +264,10 @@ private:
//get the final string result_set from SPARQLquery
bool getFinalResult(SPARQLquery& _sparql_q, ResultSet& _result_set);
static int read_update_log(const string _path, multiset<string>& _i, multiset<string>& _r);
bool restore_update(multiset<string>& _i, multiset<string>& _r);
void clear_update_log();
};
#endif //_DATABASE_DATABASE_H

597
Database/Join.cpp
View File

File diff suppressed because it is too large Load Diff

53
Database/Join.h
View File

@ -15,23 +15,23 @@
#include "../KVstore/KVstore.h"
#include "../Util/Util.h"
typedef vector<int> RecordType;
typedef vector<int>::iterator RecordIterator;
typedef vector<unsigned> RecordType;
typedef vector<unsigned>::iterator RecordIterator;
typedef list<RecordType> TableType;
typedef list<RecordType>::iterator TableIterator;
typedef list<RecordType>::reverse_iterator TableReverseIterator;
//typedef list< vector<int> > TableType;
//typedef list< vector<int> >::iterator TableIterator;
//typedef list< vector<int> >::reverse_iterator TableReverseIterator;
typedef vector< vector<int*> > IdLists;
typedef vector< vector<int> > IdListsLen;
//typedef vector< vector<int*> > IdLists;
//typedef vector< vector<int> > IdListsLen;
typedef struct Satellite
{
int id;
int* idlist;
int idlist_len;
Satellite(int _id, int* _idlist, int _idlist_len)
unsigned* idlist;
unsigned idlist_len;
Satellite(int _id, unsigned* _idlist, unsigned _idlist_len)
{
this->id = _id;
this->idlist = _idlist;
@ -47,20 +47,22 @@ private:
int var_num;
BasicQuery* basic_query;
KVstore* kvstore;
TNUM* pre2num;
int limitID_predicate;
int limitID_literal;
TYPE_TRIPLE_NUM* pre2num;
TYPE_PREDICATE_ID limitID_predicate;
TYPE_ENTITY_LITERAL_ID limitID_literal;
TYPE_ENTITY_LITERAL_ID limitID_entity;
//used by score_node for parameters
static const unsigned PARAM_DEGREE = 1;
static const unsigned PARAM_SIZE = 1000000;
static const unsigned PARAM_PRE = 10000;
static const unsigned PARAM_DENSE = 1;
//constexpr needed in C++11 (or in newer g++)
//static const double JUDGE_LIMIT = 0.5;
static const int JUDGE_LIMIT = 2; //use 1.0/JUDGE_LIMIT
static const int LIMIT_CANDIDATE_LIST_SIZE = 1000;
static const unsigned JUDGE_LIMIT = 2;
//NOTICE+DEBUG: please use constexpr below instead of the phase above(constexpr is supported in C++11)
//http://www.cnblogs.com/wanyuanchun/p/4041080.html
//constexpr static const double JUDGE_LIMIT = 0.5;
static const unsigned LIMIT_CANDIDATE_LIST_SIZE = 1000;
//BETTER?:predefine size to avoid copy cost
TableType current_table;
TableIterator new_start; //keep to end() as default
@ -73,10 +75,10 @@ private:
bool* dealed_triple;
stack<int> mystack;
vector<int*>* result_list;
vector<unsigned*>* result_list;
vector<Satellite> satellites;
int* record;
int record_len;
unsigned* record;
unsigned record_len;
void init(BasicQuery* _basic_query);
void clear();
@ -85,7 +87,7 @@ private:
//judge which method should be used according to
//the size of candidates and structure of quering graph
int judge(int _smallest, int _biggest);
int judge(unsigned _smallest, unsigned _biggest);
//select the start point and search order
void select();
@ -102,6 +104,8 @@ private:
bool filter_before_join();
bool constant_edge_filter(int _var_i);
void preid_filter(int _var_i);
//new
bool pre_handler();
bool only_pre_filter_after_join();
void add_literal_candidate();
bool pre_var_handler();
@ -112,20 +116,20 @@ private:
//functions for help
//copy/add to the end of current_table and set true
void add_new_to_results(TableIterator it, int id);
void add_new_to_results(TableIterator it, unsigned id);
//void set_results_old(list<bool>::iterator it);
int choose_next_node(int id);
bool is_literal_var(int id);
bool is_literal_ele(int _id);
//bool is_literal_ele(int _id);
void copyToResult();
//BETTER?:change these params to members in class
void acquire_all_id_lists(IdLists& _id_lists, IdListsLen& _id_lists_len, IDList& _can_list, vector<int>& _edges, int _id, int _can_list_size);
void update_answer_list(IDList*& valid_ans_list, IDList& _can_list, int* id_list, int id_list_len, bool _is_literal);
bool join_two(vector< vector<int> >& _edges, IDList& _can_list, int _can_list_size, int _id, bool _is_literal);
//void acquire_all_id_lists(IdLists& _id_lists, IdListsLen& _id_lists_len, IDList& _can_list, vector<int>& _edges, int _id, unsigned _can_list_size);
void update_answer_list(IDList*& valid_ans_list, IDList& _can_list, unsigned* id_list, unsigned id_list_len, bool _is_literal);
bool join_two(vector< vector<int> >& _edges, IDList& _can_list, unsigned _can_list_size, int _id, bool _is_ready);
bool multi_join();
//NOTICE:this is only used to join a BasicQuery
@ -133,7 +137,8 @@ private:
public:
Join();
Join(KVstore* _kvstore, TNUM* _pre2num, int _limitID_predicate, int _limitID_literal);
Join(KVstore* _kvstore, TYPE_TRIPLE_NUM* _pre2num, TYPE_PREDICATE_ID _limitID_predicate, TYPE_ENTITY_LITERAL_ID _limitID_literal,
TYPE_ENTITY_LITERAL_ID _limitID_entity);
//these functions can be called by Database
bool join_sparql(SPARQLquery& _sparql_query);
bool join_basic(BasicQuery* _basic_query);

164
Database/Strategy.cpp
View File

@ -18,7 +18,7 @@ Strategy::Strategy()
//this->prepare_handler();
}
Strategy::Strategy(KVstore* _kvstore, VSTree* _vstree, TNUM* _pre2num, int _limitID_predicate, int _limitID_literal)
Strategy::Strategy(KVstore* _kvstore, VSTree* _vstree, TYPE_TRIPLE_NUM* _pre2num, TYPE_PREDICATE_ID _limitID_predicate, TYPE_ENTITY_LITERAL_ID _limitID_literal,TYPE_ENTITY_LITERAL_ID _limitID_entity)
{
this->method = 0;
this->kvstore = _kvstore;
@ -26,6 +26,8 @@ Strategy::Strategy(KVstore* _kvstore, VSTree* _vstree, TNUM* _pre2num, int _limi
this->pre2num = _pre2num;
this->limitID_predicate = _limitID_predicate;
this->limitID_literal = _limitID_literal;
this->limitID_entity = _limitID_entity;
//this->prepare_handler();
}
@ -45,7 +47,7 @@ Strategy::~Strategy()
//however, this can be dealed due to several basic queries and linking
bool
Strategy::handle(SPARQLquery& _query, ResultFilter* _result_filter)
Strategy::handle(SPARQLquery& _query)
{
#ifdef MULTI_INDEX
Util::logging("IN GeneralEvaluation::handle");
@ -58,7 +60,7 @@ Strategy::handle(SPARQLquery& _query, ResultFilter* _result_filter)
{
this->method = -1;
vector<int*>& result_list = (*iter)->getResultList();
vector<unsigned*>& result_list = (*iter)->getResultList();
//int select_var_num = (*iter)->getSelectVarNum();
//the num of vars needing to be joined, i.e. selectVarNum if only one triple
int varNum = (*iter)->getVarNum();
@ -115,7 +117,7 @@ Strategy::handle(SPARQLquery& _query, ResultFilter* _result_filter)
//BETTER: use function pointer array in C++ class
case 0:
//default:filter by vstree and then verified by join
this->handler0(*iter, result_list, _result_filter);
this->handler0(*iter, result_list);
break;
case 1:
this->handler1(*iter, result_list);
@ -140,10 +142,11 @@ Strategy::handle(SPARQLquery& _query, ResultFilter* _result_filter)
}
#else
cout << "this BasicQuery use original query strategy" << endl;
long tv_handle = Util::get_cur_time();
(this->vstree)->retrieve(_query);
//VSTREE:
//long tv_handle = Util::get_cur_time();
//(this->vstree)->retrieve(_query);
//cout << "after Retrieve, used " << (tv_retrieve - tv_handle) << "ms." << endl;
long tv_retrieve = Util::get_cur_time();
cout << "after Retrieve, used " << (tv_retrieve - tv_handle) << "ms." << endl;
this->join = new Join(kvstore, pre2num, this->limitID_predicate, this->limitID_literal);
this->join->join_sparql(_query);
@ -157,7 +160,7 @@ Strategy::handle(SPARQLquery& _query, ResultFilter* _result_filter)
}
void
Strategy::handler0(BasicQuery* _bq, vector<int*>& _result_list, ResultFilter* _result_filter)
Strategy::handler0(BasicQuery* _bq, vector<unsigned*>& _result_list)
{
//long before_filter = Util::get_cur_time();
cout << "this BasicQuery use query strategy 0" << endl;
@ -176,37 +179,33 @@ Strategy::handler0(BasicQuery* _bq, vector<int*>& _result_list, ResultFilter* _r
long tv_handle = Util::get_cur_time();
int varNum = _bq->getVarNum(); //the num of vars needing to be joined
//TODO:parallel by pthread
for (int i = 0; i < varNum; ++i)
{
if (_bq->if_need_retrieve(i) == false)
continue;
bool flag = _bq->isLiteralVariable(i);
const EntityBitSet& entityBitSet = _bq->getVarBitSet(i);
IDList* idListPtr = &(_bq->getCandidateList(i));
this->vstree->retrieveEntity(entityBitSet, idListPtr);
if (!flag)
{
//TODO:parallel by pthread, requiring that index is parallelable
//for (int i = 0; i < varNum; ++i)
//{
//if (_bq->if_need_retrieve(i) == false)
//continue;
//bool flag = _bq->isLiteralVariable(i);
//const EntityBitSet& entityBitSet = _bq->getVarBitSet(i);
//IDList* idListPtr = &(_bq->getCandidateList(i));
//this->vstree->retrieveEntity(entityBitSet, idListPtr);
//if (!flag)
//{
//cout<<"set ready: "<<i<<endl;
_bq->setReady(i);
}
//_bq->setReady(i);
//}
//the basic query should end if one non-literal var has no candidates
if (idListPtr->size() == 0 && !flag)
{
break;
}
}
//if (idListPtr->size() == 0 && !flag)
//{
//break;
//}
//}
//BETTER:end directly if one is empty!
long tv_retrieve = Util::get_cur_time();
cout << "after Retrieve, used " << (tv_retrieve - tv_handle) << "ms." << endl;
//between retrieve and join
if (_result_filter != NULL)
_result_filter->candFilterWithResultHashTable(*_bq);
Join *join = new Join(kvstore, pre2num, this->limitID_predicate, this->limitID_literal);
Join *join = new Join(kvstore, pre2num, this->limitID_predicate, this->limitID_literal,this->limitID_entity);
join->join_basic(_bq);
delete join;
@ -215,7 +214,7 @@ Strategy::handler0(BasicQuery* _bq, vector<int*>& _result_list, ResultFilter* _r
}
void
Strategy::handler1(BasicQuery* _bq, vector<int*>& _result_list)
Strategy::handler1(BasicQuery* _bq, vector<unsigned*>& _result_list)
{
long before_filter = Util::get_cur_time();
cout << "this BasicQuery use query strategy 1" << endl;
@ -223,14 +222,15 @@ Strategy::handler1(BasicQuery* _bq, vector<int*>& _result_list)
char edge_type = _bq->getEdgeType(0, 0);
int triple_id = _bq->getEdgeID(0, 0);
Triple triple = _bq->getTriple(triple_id);
int pre_id = _bq->getEdgePreID(0, 0);
int* id_list = NULL;
int id_list_len = 0;
TYPE_PREDICATE_ID pre_id = _bq->getEdgePreID(0, 0);
unsigned* id_list = NULL;
unsigned id_list_len = 0;
if (edge_type == Util::EDGE_OUT)
{
//cout<<"edge out!!!"<<endl;
int nid = (this->kvstore)->getIDByEntity(triple.object);
if (nid == -1)
TYPE_ENTITY_LITERAL_ID nid = (this->kvstore)->getIDByEntity(triple.object);
//if (nid == -1)
if (nid == INVALID_ENTITY_LITERAL_ID)
{
nid = (this->kvstore)->getIDByLiteral(triple.object);
}
@ -246,9 +246,9 @@ Strategy::handler1(BasicQuery* _bq, vector<int*>& _result_list)
cout << "after filter, used " << (after_filter - before_filter) << "ms" << endl;
_result_list.clear();
//cout<<"now to copy result to list"<<endl;
for (int i = 0; i < id_list_len; ++i)
for (unsigned i = 0; i < id_list_len; ++i)
{
int* record = new int[1]; //only this var is selected
unsigned* record = new unsigned[1]; //only this var is selected
record[0] = id_list[i];
//cout<<this->kvstore->getEntityByID(record[0])<<endl;
_result_list.push_back(record);
@ -260,20 +260,20 @@ Strategy::handler1(BasicQuery* _bq, vector<int*>& _result_list)
}
void
Strategy::handler2(BasicQuery* _bq, vector<int*>& _result_list)
Strategy::handler2(BasicQuery* _bq, vector<unsigned*>& _result_list)
{
long before_filter = Util::get_cur_time();
cout << "this BasicQuery use query strategy 2" << endl;
int triple_id = _bq->getEdgeID(0, 0);
Triple triple = _bq->getTriple(triple_id);
int pre_id = _bq->getEdgePreID(0, 0);
TYPE_PREDICATE_ID pre_id = _bq->getEdgePreID(0, 0);
//NOTICE:it is ok for var1 or var2 to be -1, i.e. not encoded
int var1_id = _bq->getIDByVarName(triple.subject);
int var2_id = _bq->getIDByVarName(triple.object);
int* id_list = NULL;
int id_list_len = 0;
unsigned* id_list = NULL;
unsigned id_list_len = 0;
if (var1_id == 0) //subject var selected
{
//use p2s directly
@ -291,9 +291,9 @@ Strategy::handler2(BasicQuery* _bq, vector<int*>& _result_list)
long after_filter = Util::get_cur_time();
cout << "after filter, used " << (after_filter - before_filter) << "ms" << endl;
_result_list.clear();
for (int i = 0; i < id_list_len; ++i)
for (unsigned i = 0; i < id_list_len; ++i)
{
int* record = new int[1]; //only one var
unsigned* record = new unsigned[1]; //only one var
record[0] = id_list[i];
_result_list.push_back(record);
}
@ -304,15 +304,15 @@ Strategy::handler2(BasicQuery* _bq, vector<int*>& _result_list)
}
void
Strategy::handler3(BasicQuery* _bq, vector<int*>& _result_list)
Strategy::handler3(BasicQuery* _bq, vector<unsigned*>& _result_list)
{
long before_filter = Util::get_cur_time();
cout << "this BasicQuery use query strategy 3" << endl;
int triple_id = _bq->getEdgeID(0, 0);
Triple triple = _bq->getTriple(triple_id);
int pre_id = _bq->getEdgePreID(0, 0);
int* id_list = NULL;
int id_list_len = 0;
TYPE_PREDICATE_ID pre_id = _bq->getEdgePreID(0, 0);
unsigned* id_list = NULL;
unsigned id_list_len = 0;
_result_list.clear();
this->kvstore->getsubIDobjIDlistBypreID(pre_id, id_list, id_list_len);
@ -328,9 +328,9 @@ Strategy::handler3(BasicQuery* _bq, vector<int*>& _result_list)
long after_filter = Util::get_cur_time();
cout << "after filter, used " << (after_filter - before_filter) << "ms" << endl;
for (int i = 0; i < id_list_len; i += 2)
for (unsigned i = 0; i < id_list_len; i += 2)
{
int* record = new int[2]; //2 vars and selected
unsigned* record = new unsigned[2]; //2 vars and selected
record[var1_id] = id_list[i];
record[var2_id] = id_list[i + 1];
_result_list.push_back(record);
@ -343,7 +343,7 @@ Strategy::handler3(BasicQuery* _bq, vector<int*>& _result_list)
}
void
Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
Strategy::handler4(BasicQuery* _bq, vector<unsigned*>& _result_list)
{
cout<<"Special Case: consider pre var in this triple"<<endl;
int varNum = _bq->getVarNum();
@ -354,8 +354,9 @@ Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
int selected_var_num = _bq->getSelectVarNum();
Triple triple = _bq->getTriple(0);
int pvpos = _bq->getSelectedPreVarPosition(triple.predicate);
int* id_list = NULL;
int id_list_len = 0;
unsigned* id_list = NULL;
unsigned id_list_len = 0;
_result_list.clear();
//cout<<"total num: "<<total_num <<endl;
@ -369,9 +370,9 @@ Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
cout<<"predicate: "<<triple.predicate<<" "<<pvpos<<endl;
//very special case, to find all triples, select ?s (?p) ?o where { ?s ?p ?o . }
//filter and join is too costly, should enum all predicates and use p2so
for(int i = 0; i < this->limitID_predicate; ++i)
for(TYPE_PREDICATE_ID i = 0; i < this->limitID_predicate; ++i)
{
int pid = i;
TYPE_PREDICATE_ID pid = i;
this->kvstore->getsubIDobjIDlistBypreID(pid, id_list, id_list_len);
int rsize = selected_var_num;
if(selected_pre_var_num == 1)
@ -380,9 +381,9 @@ Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
}
//always place s/o before p in result list
for (int j = 0; j < id_list_len; j += 2)
for (unsigned j = 0; j < id_list_len; j += 2)
{
int* record = new int[rsize];
unsigned* record = new unsigned[rsize];
//check the s/o var if selected, need to ensure the placement order
if(ovpos >= 0)
{
@ -409,14 +410,15 @@ Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
int vpos = -1;
if(triple.subject[0] != '?') //constant
{
int sid = (this->kvstore)->getIDByEntity(triple.subject);
TYPE_ENTITY_LITERAL_ID sid = (this->kvstore)->getIDByEntity(triple.subject);
this->kvstore->getpreIDobjIDlistBysubID(sid, id_list, id_list_len);
vpos = _bq->getSelectedVarPosition(triple.object);
}
else if(triple.object[0] != '?') //constant
{
int oid = (this->kvstore)->getIDByEntity(triple.object);
if (oid == -1)
TYPE_ENTITY_LITERAL_ID oid = (this->kvstore)->getIDByEntity(triple.object);
//if (oid == -1)
if (oid == INVALID_ENTITY_LITERAL_ID)
{
oid = (this->kvstore)->getIDByLiteral(triple.object);
}
@ -430,9 +432,9 @@ Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
rsize++;
}
//always place s/o before p in result list
for (int i = 0; i < id_list_len; i += 2)
for (unsigned i = 0; i < id_list_len; i += 2)
{
int* record = new int[rsize];
unsigned* record = new unsigned[rsize];
if(vpos >= 0)
{
record[vpos] = id_list[i + 1]; //for the s/o var
@ -448,8 +450,8 @@ Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
{
cout<<"Special Case 3"<<endl;
//just use so2p
int sid = (this->kvstore)->getIDByEntity(triple.subject);
int oid = (this->kvstore)->getIDByEntity(triple.object);
unsigned sid = (this->kvstore)->getIDByEntity(triple.subject);
unsigned oid = (this->kvstore)->getIDByEntity(triple.object);
if (oid == -1)
{
oid = (this->kvstore)->getIDByLiteral(triple.object);
@ -457,9 +459,9 @@ Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
this->kvstore->getpreIDlistBysubIDobjID(sid, oid, id_list, id_list_len);
//copy to result list
for (int i = 0; i < id_list_len; ++i)
for (unsigned i = 0; i < id_list_len; ++i)
{
int* record = new int[1];
unsigned* record = new unsigned[1];
record[0] = id_list[i];
_result_list.push_back(record);
}
@ -471,38 +473,42 @@ Strategy::handler4(BasicQuery* _bq, vector<int*>& _result_list)
//TODO:if any constants in a query are not found in kvstore, then this BGP should end to speed up the processing
void
Strategy::handler5(BasicQuery* _bq, vector<int*>& _result_list)
Strategy::handler5(BasicQuery* _bq, vector<unsigned*>& _result_list)
{
cout<<"Special Case: consider constant triple"<<endl;
Triple triple = _bq->getTriple(0);
_result_list.clear();
int subid = this->kvstore->getIDByEntity(triple.subject);
if(subid == -1) //not found
TYPE_ENTITY_LITERAL_ID subid = this->kvstore->getIDByEntity(triple.subject);
//if(subid == -1) //not found
if(subid == INVALID_ENTITY_LITERAL_ID) //not found
{
return;
}
int preid = this->kvstore->getIDByPredicate(triple.predicate);
if(preid == -1) //not found
TYPE_PREDICATE_ID preid = this->kvstore->getIDByPredicate(triple.predicate);
//if(preid == -1) //not found
if(preid == INVALID_PREDICATE_ID) //not found
{
return;
}
int objid = this->kvstore->getIDByEntity(triple.object);
if(objid == -1)
TYPE_ENTITY_LITERAL_ID objid = this->kvstore->getIDByEntity(triple.object);
//if(objid == -1)
if(objid == INVALID_ENTITY_LITERAL_ID)
{
objid = this->kvstore->getIDByLiteral(triple.object);
}
if(objid == -1)
//if(objid == -1)
if(objid == INVALID_ENTITY_LITERAL_ID)
{
return;
}
int* id_list = NULL;
int id_list_len = 0;
unsigned* id_list = NULL;
unsigned id_list_len = 0;
(this->kvstore)->getobjIDlistBysubIDpreID(subid, preid, id_list, id_list_len);
if (Util::bsearch_int_uporder(objid, id_list, id_list_len) != -1)
if (Util::bsearch_int_uporder(objid, id_list, id_list_len) != INVALID)
{
int* record = new int[3];
unsigned* record = new unsigned[3];
record[0] = subid;
record[1] = preid;
record[2] = objid;

29
Database/Strategy.h
View File

@ -17,37 +17,40 @@
#include "../Query/BasicQuery.h"
#include "../KVstore/KVstore.h"
#include "../VSTree/VSTree.h"
#include "../Query/ResultFilter.h"
class Strategy
{
public:
Strategy();
Strategy(KVstore*, VSTree*, TNUM*, int, int);
Strategy(KVstore*, VSTree*, TYPE_TRIPLE_NUM*, TYPE_PREDICATE_ID, TYPE_ENTITY_LITERAL_ID,TYPE_ENTITY_LITERAL_ID);
~Strategy();
//select efficient strategy to do the sparql query
bool handle(SPARQLquery&, ResultFilter* _result_filter = NULL);
bool handle(SPARQLquery&);
private:
int method;
KVstore* kvstore;
VSTree* vstree;
TNUM* pre2num;
int limitID_predicate;
int limitID_literal;
TYPE_TRIPLE_NUM* pre2num;
TYPE_PREDICATE_ID limitID_predicate;
TYPE_ENTITY_LITERAL_ID limitID_literal;
TYPE_ENTITY_LITERAL_ID limitID_entity;
void handler0(BasicQuery*, vector<int*>&, ResultFilter* _result_filter = NULL);
void handler1(BasicQuery*, vector<int*>&);
void handler2(BasicQuery*, vector<int*>&);
void handler3(BasicQuery*, vector<int*>&);
void handler4(BasicQuery*, vector<int*>&);
void handler5(BasicQuery*, vector<int*>&);
//NOTICE: even the ID type is int, it is no problem and no waste that we use unsigned in answer
//(because -1, -2 or other invalid IDs can not be in answer)
void handler0(BasicQuery*, vector<unsigned*>&);
void handler1(BasicQuery*, vector<unsigned*>&);
void handler2(BasicQuery*, vector<unsigned*>&);
void handler3(BasicQuery*, vector<unsigned*>&);
void handler4(BasicQuery*, vector<unsigned*>&);
void handler5(BasicQuery*, vector<unsigned*>&);
//QueryHandler *dispatch;
//void prepare_handler();
};
//function pointer array
static const unsigned QUERY_HANDLER_NUM = 4;
typedef void (Strategy::*QueryHandler[QUERY_HANDLER_NUM]) (BasicQuery*, vector<int*>&);
typedef void (Strategy::*QueryHandler[QUERY_HANDLER_NUM]) (BasicQuery*, vector<unsigned*>&);
//QueryHandler dispatch;
#endif //_DATABASE_STRATEGY_H

170
KVstore/ISTree/ISTree.cpp
View File

@ -20,7 +20,7 @@ ISTree::ISTree()
TSM = NULL;
storepath = "";
filename = "";
transfer_size[0] = transfer_size[1] = transfer_size[2] = 0;
//transfer_size[0] = transfer_size[1] = transfer_size[2] = 0;
this->stream = NULL;
this->request = 0;
}
@ -37,10 +37,10 @@ ISTree::ISTree(string _storepath, string _filename, string _mode, unsigned long
this->TSM->preRead(this->root, this->leaves_head, this->leaves_tail);
else
this->root = NULL;
this->transfer[0].setStr((char*)malloc(Util::TRANSFER_SIZE));
this->transfer[1].setStr((char*)malloc(Util::TRANSFER_SIZE));
this->transfer[2].setStr((char*)malloc(Util::TRANSFER_SIZE));
this->transfer_size[0] = this->transfer_size[1] = this->transfer_size[2] = Util::TRANSFER_SIZE; //initialied to 1M
//this->transfer[0].setStr((char*)malloc(Util::TRANSFER_SIZE));
//this->transfer[1].setStr((char*)malloc(Util::TRANSFER_SIZE));
//this->transfer[2].setStr((char*)malloc(Util::TRANSFER_SIZE));
//this->transfer_size[0] = this->transfer_size[1] = this->transfer_size[2] = Util::TRANSFER_SIZE; //initialied to 1M
this->stream = NULL;
this->request = 0;
}
@ -51,30 +51,30 @@ ISTree::getFilePath()
return storepath + "/" + filename;
}
void //WARN: not check _str and _len
ISTree::CopyToTransfer(const char* _str, unsigned _len, unsigned _index)
{
if (_index > 2)
return;
/*
if(_str == NULL || _len == 0)
{
printf("error in CopyToTransfer: empty string\n");
return;
}
*/
//unsigned length = _bstr->getLen();
unsigned length = _len;
if (length + 1 > this->transfer_size[_index])
{
transfer[_index].release();
transfer[_index].setStr((char*)malloc(length + 1));
this->transfer_size[_index] = length + 1; //one more byte: convenient to add \0
}
memcpy(this->transfer[_index].getStr(), _str, length);
this->transfer[_index].getStr()[length] = '\0'; //set for string() in KVstore
this->transfer[_index].setLen(length);
}
//void //WARN: not check _str and _len
//ISTree::CopyToTransfer(const char* _str, unsigned _len, unsigned _index)
//{
//if (_index > 2)
//return;
//[>
//if(_str == NULL || _len == 0)
//{
//printf("error in CopyToTransfer: empty string\n");
//return;
//}
//*/
////unsigned length = _bstr->getLen();
//unsigned length = _len;
//if (length + 1 > this->transfer_size[_index])
//{
//transfer[_index].release();
//transfer[_index].setStr((char*)malloc(length + 1));
//this->transfer_size[_index] = length + 1; //one more byte: convenient to add \0
//}
//memcpy(this->transfer[_index].getStr(), _str, length);
//this->transfer[_index].getStr()[length] = '\0'; //set for string() in KVstore
//this->transfer[_index].setLen(length);
//}
unsigned
ISTree::getHeight() const
@ -104,41 +104,59 @@ ISTree::prepare(ISNode* _np)
}
bool
ISTree::search(int _key, char*& _str, int& _len)
ISTree::search(unsigned _key, char*& _str, unsigned& _len)
{
if (_key < 0)
{
printf("error in ISTree-search: empty string\n");
return false;
}
//DEBUG
//if (_key < 0)
//{
//printf("error in ISTree-search: empty string\n");
//return false;
//}
this->request = 0;
int store;
ISNode* ret = this->find(_key, &store, false);
//cout<<"to find the position: "<<store<<endl;
if (ret == NULL || store == -1 || _key != ret->getKey(store)) //tree is empty or not found
{
return false;
}
const Bstr* val = ret->getValue(store);
this->CopyToTransfer(val->getStr(), val->getLen(), 0); //not sum to request
_str = this->transfer[0].getStr();
_len = this->transfer[0].getLen();
//this->CopyToTransfer(val->getStr(), val->getLen(), 0); //not sum to request
//_str = this->transfer[0].getStr();
//_len = this->transfer[0].getLen();
_str = val->getStr();
_len = val->getLen();
char* debug = new char[_len];
memcpy(debug, _str, _len);
// debug[_len] = '\0';
_str = debug;
//if(_key==62)
//{
//cout<<"check in search: "<<string(_str, _len)<<endl;
//}
this->TSM->request(request);
//if(_key==62)
//{
//cout<<"check in search: "<<string(_str, _len)<<endl;
//}
return true;
}
bool
ISTree::insert(int _key, const char* _str, unsigned _len)
ISTree::insert(unsigned _key, char* _str, unsigned _len)
{
if (_key < 0)
{
printf("error in ISTree-insert: empty string\n");
return false;
}
//if (_key < 0)
//{
//printf("error in ISTree-insert: empty string\n");
//return false;
//}
this->CopyToTransfer(_str, _len, 2);
const Bstr* val = &(this->transfer[2]);
//this->CopyToTransfer(_str, _len, 2);
//const Bstr* val = &(this->transfer[2]);
this->request = 0;
ISNode* ret;
if (this->root == NULL) //tree is empty
@ -222,29 +240,33 @@ ISTree::insert(int _key, const char* _str, unsigned _len)
else
{
p->addKey(_key, i);
p->addValue(val, i, true);
p->addValue(_str, _len, i, true);
p->addNum();
request += val->getLen();
request += _len;
p->setDirty();
this->TSM->updateHeap(p, p->getRank(), true);
//_key->clear();
//_value->clear();
}
this->TSM->request(request);
//if(_key == 0)
//{
//cout<<"the 0th element is: "<<_str[0]<<endl;
//}
return !ifexist; //QUERY(which case:return false)
}
bool
ISTree::modify(int _key, const char* _str, unsigned _len)
ISTree::modify(unsigned _key, char* _str, unsigned _len)
{
if (_key < 0)
{
printf("error in ISTree-modify: empty string\n");
return false;
}
//if (_key < 0)
//{
//printf("error in ISTree-modify: empty string\n");
//return false;
//}
this->CopyToTransfer(_str, _len, 2); //not check value
const Bstr* val = &(this->transfer[2]);
//this->CopyToTransfer(_str, _len, 2); //not check value
//const Bstr* val = &(this->transfer[2]);
this->request = 0;
int store;
ISNode* ret = this->find(_key, &store, true);
@ -255,22 +277,23 @@ ISTree::modify(int _key, const char* _str, unsigned _len)
}
//cout<<"ISTree::modify() - key is found, now to remove"<<endl;
unsigned len = ret->getValue(store)->getLen();
ret->setValue(val, store, true);
ret->setValue(_str, _len, store, true);
//cout<<"value reset"<<endl;
//cout<<"newlen: "<<val->getLen()<<" oldlen: "<<len<<endl;
//request += (val->getLen() - len);
this->request = val->getLen();
this->request = _len;
this->request -= len;
ret->setDirty();
//cout<<"to request"<<endl;
this->TSM->request(request);
//cout<<"memory requested"<<endl;
return true;
}
//this function is useful for search and modify, and range-query
ISNode* //return the first key's position that >= *_key
ISTree::find(int _key, int* _store, bool ifmodify)
ISTree::find(unsigned _key, int* _store, bool ifmodify)
{ //to assign value for this->bstr, function shouldn't be const!
if (this->root == NULL)
return NULL; //ISTree Is Empty
@ -300,6 +323,7 @@ ISTree::find(int _key, int* _store, bool ifmodify)
*_store = -1; //Not Found
else
*_store = i;
return p;
}
@ -311,13 +335,14 @@ ISTree::find(unsigned _len, const char* _str, int* store) const
*/
bool
ISTree::remove(int _key)
ISTree::remove(unsigned _key)
{
if (_key < 0)
{
printf("error in ISTree-remove: empty string\n");
return false;
}
//DEBUG
//if (_key < 0)
//{
//printf("error in ISTree-remove: empty string\n");
//return false;
//}
this->request = 0;
ISNode* ret;
@ -443,7 +468,7 @@ ISTree::resetStream()
}
bool //special case: not exist, one-edge-case
ISTree::range_query(int _key1, int _key2)
ISTree::range_query(unsigned _key1, unsigned _key2)
{ //the range is: *_key1 <= x < *_key2
//if(_key1 <0 && _key2 <0)
//return false;
@ -516,7 +541,7 @@ ISTree::range_query(int _key1, int _key2)
delete this->stream;
this->stream = NULL;
}
vector<int> keys;
vector<unsigned> keys;
vector<bool> desc;
this->stream = new Stream(keys, desc, ansNum, 1, false);
@ -570,6 +595,7 @@ ISTree::release(ISNode* _np) const
return;
}
int cnt = _np->getNum();
//WARN: not chnage cnt to int type here(otherwise endless loop)
for (; cnt >= 0; --cnt)
release(_np->getChild(cnt));
delete _np;
@ -577,13 +603,20 @@ ISTree::release(ISNode* _np) const
ISTree::~ISTree()
{
//cout << "istree : " << endl;
//cout << "delete stream" << endl;
delete this->stream; //maybe NULL
stream = NULL;
//cout << "delete TSM" << endl;
delete TSM;
TSM = NULL;
#ifdef DEBUG_KVSTORE
printf("already empty the buffer, now to delete all nodes in tree!\n");
//printf("already empty the buffer, now to delete all nodes in tree!\n");
#endif
//recursively delete each Node
//cout << "release" << endl;
release(root);
//cout << "~istree done" << endl;
}
void
@ -655,3 +688,4 @@ ISTree::print(string s)
else;
#endif
}

27
KVstore/ISTree/ISTree.h
View File

@ -3,7 +3,7 @@
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:44
# Description: struct and interface of the B+ tree
# Description: ID2string, including id2entity, id2literal and id2predicate
=============================================================================*/
#ifndef _KVSTORE_ISTREE_ISTREE_H
@ -19,7 +19,7 @@
class ISTree
{
protected:
unsigned int height; //0 indicates an empty tree
unsigned height; //0 indicates an empty tree
ISNode* root;
ISNode* leaves_head; //the head of LeafNode-list
ISNode* leaves_tail; //the tail of LeafNode-list
@ -36,8 +36,8 @@ protected:
//so lock is a must. Add lock to transfer is better than to add
//lock to every key/value. However, modify requires a lock for a
//key/value, and multiple search for different keys are ok!!!
Bstr transfer[3]; //0:transfer value searched; 1:copy key-data from const char*; 2:copy val-data from const char*
unsigned transfer_size[3];
//Bstr transfer[3]; //0:transfer value searched; 1:copy key-data from const char*; 2:copy val-data from const char*
//unsigned transfer_size[3];
//tree's operations should be atom(if read nodes)
//sum the request and send to ISStorage at last
@ -49,25 +49,25 @@ protected:
std::string filename; //ok for user to change
/* some private functions */
std::string getFilePath(); //in UNIX system
void CopyToTransfer(const char* _str, unsigned _len, unsigned _index);
//void CopyToTransfer(const char* _str, unsigned _len, unsigned _index);
void release(ISNode* _np) const;
public:
ISTree(); //always need to initial transfer
ISTree(std::string _storepath, std::string _filename, std::string _mode, unsigned long long _buffer_size);
unsigned int getHeight() const;
unsigned getHeight() const;
void setHeight(unsigned _h);
ISNode* getRoot() const;
//void setRoot(Node* _root);
//insert, search, remove, set
bool search(int _key, char*& _str, int& _len);
bool insert(int _key, const char* _str, unsigned _len);
bool modify(int _key, const char* _str, unsigned _len);
ISNode* find(int _key, int* store, bool ifmodify);
bool remove(int _key);
bool search(unsigned _key, char*& _str, unsigned& _len);
bool insert(unsigned _key, char* _str, unsigned _len);
bool modify(unsigned _key, char* _str, unsigned _len);
ISNode* find(unsigned _key, int* store, bool ifmodify);
bool remove(unsigned _key);
const Bstr* getRangeValue();
void resetStream();
bool range_query(int _key1, int _key2);
bool range_query(unsigned _key1, unsigned _key2);
bool save();
~ISTree();
void print(std::string s); //DEBUG(print the tree)
@ -76,4 +76,5 @@ public:
//(problem range between two extremes: not-modified, totally-modified)
//After saved, it's ok to continue operations on tree!
#endif
#endif

13
KVstore/ISTree/heap/ISHeap.cpp
View File

@ -20,8 +20,8 @@ ISHeap::ISHeap(unsigned _size)
{
this->length = 0;
this->size = _size;
//this->heap = (Node**)malloc(this->size * sizeof(Node*)); //not use 4 or 8
this->heap = new ISNode*[this->size];
this->heap = (ISNode**)malloc(this->size * sizeof(ISNode*)); //not use 4 or 8
//this->heap = new ISNode*[this->size];
if (this->heap == NULL)
{
this->print("error in ISHeap: Allocation fail!");
@ -69,6 +69,10 @@ ISHeap::insert(ISNode* _np)
{
if (this->length == this->size) //when full, reallocate
{
cout<<"check: double the heap"<<endl;
//WARN: realloc should be combined with malloc instead of new
//http://bbs.csdn.net/topics/320148799
//For new, use placement new to enlarge space: http://blog.csdn.net/vangoals/article/details/4252833
this->heap = (ISNode**)realloc(this->heap, 2 * this->size * sizeof(ISNode*));
if (this->heap == NULL)
{
@ -183,7 +187,8 @@ ISHeap::modify(ISNode* _np, bool _flag) //control direction
ISHeap::~ISHeap()
{
delete[] this->heap;
//delete[] this->heap;
free(this->heap);
this->heap = NULL;
this->length = this->size = 0;
}
@ -193,4 +198,4 @@ ISHeap::print(string s)
{
#ifdef DEBUG_KVSTORE
#endif
}
}

3
KVstore/ISTree/heap/ISHeap.h
View File

@ -38,4 +38,5 @@ public:
void print(std::string s); //DEBUG
};
#endif
#endif

6
KVstore/ISTree/node/ISIntlNode.cpp
View File

@ -131,7 +131,7 @@ ISIntlNode::split(ISNode* _father, int _index)
p->addNum();
}
p->addChild(this->childs[i], k);
int tp = this->keys[MIN_KEY_NUM];
unsigned tp = this->keys[MIN_KEY_NUM];
this->setNum(MIN_KEY_NUM);
_father->addKey(tp, _index);
_father->addChild(p, _index + 1); //DEBUG(check the index)
@ -177,7 +177,7 @@ ISIntlNode::coalesce(ISNode* _father, int _index)
}
}
int tmp = 0;
unsigned tmp = 0;
switch (ccase)
{
case 1: //union right to this
@ -290,4 +290,4 @@ ISIntlNode::print(string s)
}
else;
#endif
}
}

3
KVstore/ISTree/node/ISIntlNode.h
View File

@ -45,4 +45,5 @@ public:
*/
};
#endif
#endif

55
KVstore/ISTree/node/ISLeafNode.cpp
View File

@ -82,6 +82,7 @@ ISLeafNode::getValue(int _index) const
int num = this->getNum();
if (_index < 0 || _index >= num)
{
//cout<<"null in getValue: "<<_index<<endl;
//print(string("error in getValue: Invalid index ") + Util::int2string(_index));
return NULL;
}
@ -125,13 +126,53 @@ ISLeafNode::addValue(const Bstr* _value, int _index, bool ifcopy)
return true;
}
bool
ISLeafNode::setValue(char* _str, unsigned _len, int _index, bool ifcopy)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
//print(string("error in setValue: Invalid index ") + Util::int2string(_index));
return false;
}
this->values[_index].release(); //NOTICE: only used in modify
this->values[_index].setStr(_str);
this->values[_index].setLen(_len);
return true;
}
bool
ISLeafNode::addValue(char* _str, unsigned _len, int _index, bool ifcopy)
{
int num = this->getNum();
//cout<<"addValue: "<<num<<" "<<_index<<endl;
if (_index < 0 || _index > num)
{
//print(string("error in addValue: Invalid index ") + Util::int2string(_index));
//cout<<"error in addValue: "<<_index<<" "<<num<<endl;
return false;
}
int i;
for (i = num - 1; i >= _index; --i)
this->values[i + 1] = this->values[i];
this->values[_index].setStr(_str);
this->values[_index].setLen(_len);
//cout<<"show: "<<this->values[_index].getLen()<<" "<<this->values[_index].getStr()[0]<<endl;
return true;
}
bool
ISLeafNode::subValue(int _index, bool ifdel)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
print(string("error in subValue: Invalid index ") + Util::int2string(_index));
//print(string("error in subValue: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
@ -181,7 +222,8 @@ ISLeafNode::split(ISNode* _father, int _index)
p->addValue(this->values + i, k);
p->addNum();
}
int tp = this->keys[MIN_KEY_NUM];
unsigned tp = this->keys[MIN_KEY_NUM];
//cout<<"split: "<<tp<<" "<<this->values[MIN_KEY_NUM].getStr()<<endl;
this->setNum(MIN_KEY_NUM);
_father->addKey(tp, _index);
_father->addChild(p, _index + 1); //DEBUG(check the index)
@ -226,7 +268,7 @@ ISLeafNode::coalesce(ISNode* _father, int _index)
}
}
int tmp = 0;
unsigned tmp = 0;
switch (ccase)
{
case 1: //union right to this
@ -283,8 +325,8 @@ ISLeafNode::coalesce(ISNode* _father, int _index)
p->subNum();
break;
default:
print("error in coalesce: Invalid case!");
//printf("error in coalesce: Invalid case!");
//print("error in coalesce: Invalid case!");
cout<<"error in coalesce: Invalid case!"<<endl;
}
_father->setDirty();
p->setDirty();
@ -373,4 +415,5 @@ ISLeafNode::print(string s)
}
else;
#endif
}
}

8
KVstore/ISTree/node/ISLeafNode.h
View File

@ -27,6 +27,7 @@ public:
void Normal();
ISNode* getPrev() const;
ISNode* getNext() const;
const Bstr* getValue(int _index) const;
bool setValue(const Bstr* _value, int _index, bool ifcopy = false);
bool addValue(const Bstr* _value, int _index, bool ifcopy = false);
@ -34,6 +35,10 @@ public:
void setPrev(ISNode* _prev);
void setNext(ISNode* _next);
unsigned getSize() const;
bool setValue(char* _str, unsigned _len, int _index, bool ifcopy = false);
bool addValue(char* _str, unsigned _len, int _index, bool ifcopy = false);
ISNode* split(ISNode* _father, int _index);
ISNode* coalesce(ISNode* _father, int _index);
void release();
@ -47,4 +52,5 @@ public:
};
//BETTER: prev isn't a must, and reverse-range can be achieved using recursive-next
#endif
#endif

18
KVstore/ISTree/node/ISNode.cpp
View File

@ -205,7 +205,7 @@ ISNode::setFlag(unsigned _flag)
this->flag = _flag;
}
int
unsigned
ISNode::getKey(int _index) const
{
int num = this->getNum();
@ -213,14 +213,15 @@ ISNode::getKey(int _index) const
{
//print(string("error in getKey: Invalid index ") + Util::int2string(_index));
printf("error in getKey: Invalid index\n");
return -1;
//return -1;
return INVALID;
}
else
return this->keys[_index];
}
bool
ISNode::setKey(int _key, int _index)
ISNode::setKey(unsigned _key, int _index)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
@ -233,7 +234,7 @@ ISNode::setKey(int _key, int _index)
}
bool
ISNode::addKey(int _key, int _index)
ISNode::addKey(unsigned _key, int _index)
{
int num = this->getNum();
if (_index < 0 || _index > num)
@ -266,7 +267,7 @@ ISNode::subKey(int _index)
}
int
ISNode::searchKey_less(int _key) const
ISNode::searchKey_less(unsigned _key) const
{
int num = this->getNum();
//for(i = 0; i < num; ++i)
@ -292,7 +293,7 @@ ISNode::searchKey_less(int _key) const
}
int
ISNode::searchKey_equal(int _key) const
ISNode::searchKey_equal(unsigned _key) const
{
int num = this->getNum();
//for(i = 0; i < num; ++i)
@ -307,7 +308,7 @@ ISNode::searchKey_equal(int _key) const
}
int
ISNode::searchKey_lessEqual(int _key) const
ISNode::searchKey_lessEqual(unsigned _key) const
{
//int num = this->getNum();
//for(i = 0; i < num; ++i)
@ -319,4 +320,5 @@ ISNode::searchKey_lessEqual(int _key) const
return ret - 1;
else
return ret;
}
}

21
KVstore/ISTree/node/ISNode.h
View File

@ -66,15 +66,15 @@ public:
void setStore(unsigned _store);
unsigned getFlag() const;
void setFlag(unsigned _flag);
int getKey(int _index) const; //need to check the index
bool setKey(int _key, int _index);
bool addKey(int _key, int _index);
unsigned getKey(int _index) const; //need to check the index
bool setKey(unsigned _key, int _index);
bool addKey(unsigned _key, int _index);
bool subKey(int _index);
//several binary key search utilities
int searchKey_less(int _key) const;
int searchKey_equal(int _key) const;
int searchKey_lessEqual(int _key) const;
int searchKey_less(unsigned _key) const;
int searchKey_equal(unsigned _key) const;
int searchKey_lessEqual(unsigned _key) const;
//virtual functions: polymorphic
virtual ISNode* getChild(int _index) const { return NULL; };
@ -83,12 +83,18 @@ public:
virtual bool subChild(int _index) { return true; };
virtual ISNode* getPrev() const { return NULL; };
virtual ISNode* getNext() const { return NULL; };
virtual const Bstr* getValue(int _index) const { return NULL; };
virtual bool setValue(const Bstr* _value, int _index, bool ifcopy = false) { return true; };
virtual bool addValue(const Bstr* _value, int _index, bool ifcopy = false) { return true; };
virtual bool subValue(int _index, bool ifdel = false) { return true; };
virtual void setPrev(ISNode* _prev) {};
virtual void setNext(ISNode* _next) {};
virtual bool setValue(char* _str, unsigned _len, int _index, bool ifcopy = false) { return true; };
virtual bool addValue(char* _str, unsigned _len, int _index, bool ifcopy = false) { return true; };
//pure virtual function
virtual void Virtual() = 0;
virtual void Normal() = 0;
virtual unsigned getSize() const = 0; //return all memory owned
@ -113,4 +119,5 @@ public:
*to release the whole(pointer is invalid and rebuild problem)
*/
#endif
#endif

24
KVstore/ISTree/storage/ISStorage.cpp
View File

@ -69,7 +69,7 @@ ISStorage::ISStorage(string& _filepath, string& _mode, unsigned* _height, unsign
else //_mode == "open"
{
//read basic information
int rootnum;
unsigned rootnum;
char c;
fread(this->treeheight, sizeof(unsigned), 1, this->treefp);
fread(&rootnum, sizeof(unsigned), 1, this->treefp);
@ -274,7 +274,8 @@ ISStorage::readNode(ISNode* _np, long long* _request)
fseek(treefp, 4 * (num + 1), SEEK_CUR);
//to read all keys
int tmp = -1;
unsigned tmp = INVALID;
//int tmp = -1;
for (i = 0; i < num; ++i)
{
fread(&tmp, sizeof(int), 1, treefp);
@ -384,7 +385,8 @@ ISStorage::writeNode(ISNode* _np)
}
}
int tmp = 0;
//int tmp = 0;
unsigned tmp = INVALID;
//to write all keys
for (i = 0; i < num; ++i)
{
@ -397,7 +399,13 @@ ISStorage::writeNode(ISNode* _np)
{
//to write all values
for (i = 0; i < num; ++i)
{
this->writeBstr(_np->getValue(i), &blocknum, SpecialBlock);
//if(_np->getKey(0) == 0)
//{
//cout<<"the 0th value: "<<_np->getValue(i)->getStr()[0]<<endl;
//}
}
}
fseek(treefp, Address(blocknum), SEEK_SET);
if (SpecialBlock)
@ -408,6 +416,7 @@ ISStorage::writeNode(ISNode* _np)
//NOTICE:we may store the dirty bit into the tree file, but that is ok
//Each time we read the tree file to construct a node, we always set the drity bit to 0
_np->delDirty();
return true;
}
@ -419,7 +428,8 @@ ISStorage::readBstr(Bstr* _bp, unsigned* _next)
fread(&len, sizeof(unsigned), 1, this->treefp);
this->ReadAlign(_next);
//this->request(len);
char* s = (char*)malloc(len);
//char* s = (char*)malloc(len);
char* s = new char[len];
_bp->setLen(len);
for (i = 0; i + 4 < len; i += 4)
{
@ -437,6 +447,7 @@ ISStorage::readBstr(Bstr* _bp, unsigned* _next)
fseek(treefp, j, SEEK_CUR);
this->ReadAlign(_next);
_bp->setStr(s);
return true;
}
@ -551,7 +562,10 @@ ISStorage::writeTree(ISNode* _root) //write the whole tree back and close treefp
fputc(c & ~(1 << j), treefp);
bp = bp->next;
}
fflush(this->treefp);
//fclose(this->treefp);
return true;
}
@ -653,6 +667,7 @@ ISStorage::~ISStorage()
printf("already empty the freelist!\n");
#endif
delete this->minheap;
minheap = NULL;
#ifdef DEBUG_KVSTORE
printf("already empty the buffer heap!\n");
#endif
@ -675,3 +690,4 @@ ISStorage::print(string s)
fputs("\n", Util::debug_kvstore);
#endif
}

3
KVstore/ISTree/storage/ISStorage.h
View File

@ -43,6 +43,8 @@ private:
//Because the bstr' size is controlled, so is the node.
unsigned long long freemem; //free memory to use, non-negative
//unsigned long long time; //QUERY(achieving an old-swap startegy?)
//QUERY: should this be long long? (otherwise will be different in 32-bit and 64-bit machine)
long Address(unsigned _blocknum) const;
unsigned Blocknum(long address) const;
unsigned AllocBlock();
@ -70,3 +72,4 @@ public:
};
#endif

737
KVstore/IVTree/IVTree.cpp Normal file
View File

@ -0,0 +1,737 @@
/*=============================================================================
# Filename: IVTree.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:45
# Description: achieve functions in IVTree.h
=============================================================================*/
#include "IVTree.h"
using namespace std;
IVTree::IVTree()
{
height = 0;
mode = "";
root = NULL;
leaves_head = NULL;
leaves_tail = NULL;
TSM = NULL;
storepath = "";
filename = "";
//transfer_size[0] = transfer_size[1] = transfer_size[2] = 0;
//transfer_size = 0;
this->stream = NULL;
this->request = 0;
this->value_list = NULL;
}
IVTree::IVTree(string _storepath, string _filename, string _mode, unsigned long long _buffer_size)
{
storepath = _storepath;
filename = _filename;
this->height = 0;
this->mode = string(_mode);
string filepath = this->getFilePath();
string vlist_file = filepath + "_vlist";
this->value_list = new VList(vlist_file, this->mode, 1<<30);
TSM = new IVStorage(filepath, this->mode, &this->height, _buffer_size, this->value_list);
if (this->mode == "open")
this->TSM->preRead(this->root, this->leaves_head, this->leaves_tail);
else
this->root = NULL;
//this->transfer[0].setStr((char*)malloc(Util::TRANSFER_SIZE));
//this->transfer[1].setStr((char*)malloc(Util::TRANSFER_SIZE));
//this->transfer[2].setStr((char*)malloc(Util::TRANSFER_SIZE));
//this->transfer_size[0] = this->transfer_size[1] = this->transfer_size[2] = Util::TRANSFER_SIZE; //initialied to 1M
//this->transfer.setStr((char*)malloc(Util::TRANSFER_SIZE));
this->stream = NULL;
this->request = 0;
}
string
IVTree::getFilePath()
{
return storepath + "/" + filename;
}
//void //WARN: not check _str and _len
//IVTree::CopyToTransfer(const char* _str, unsigned _len, unsigned _index)
//{
//if (_index > 2)
//return;
//[>
//if(_str == NULL || _len == 0)
//{
//printf("error in CopyToTransfer: empty string\n");
//return;
//}
//*/
////unsigned length = _bstr->getLen();
//unsigned length = _len;
//if (length + 1 > this->transfer_size[_index])
//{
//transfer[_index].release();
//transfer[_index].setStr((char*)malloc(length + 1));
//this->transfer_size[_index] = length + 1; //one more byte: convenient to add \0
//}
//memcpy(this->transfer[_index].getStr(), _str, length);
//this->transfer[_index].getStr()[length] = '\0'; //set for string() in KVstore
//this->transfer[_index].setLen(length);
//}
unsigned
IVTree::getHeight() const
{
return this->height;
}
void
IVTree::setHeight(unsigned _h)
{
this->height = _h;
}
IVNode*
IVTree::getRoot() const
{
return this->root;
}
void
IVTree::prepare(IVNode* _np)
{
//this->request = 0;
bool flag = _np->inMem();
if (!flag)
{
this->TSM->readNode(_np, &request); //readNode deal with request
}
}
bool
IVTree::search(unsigned _key, char*& _str, unsigned& _len)
{
//if (_key < 0)
//{
//printf("error in IVTree-search: empty string\n");
//return false;
//}
this->request = 0;
int store;
IVNode* ret = this->find(_key, &store, false);
if (ret == NULL || store == -1 || _key != ret->getKey(store)) //tree is empty or not found
{
return false;
}
ret->getValue(this->value_list, store, _str, _len);
//const Bstr* val = ret->getValue(store);
//this->CopyToTransfer(val->getStr(), val->getLen(), 0); //not sum to request
//_str = this->transfer[0].getStr();
//_len = this->transfer[0].getLen();
if (!VList::isLongList(_len))
{
char* debug = new char [_len];
memcpy(debug, _str, _len);
_str = debug;
}
this->TSM->request(request);
return true;
}
bool
IVTree::insert(unsigned _key, char* _str, unsigned _len)
{
//if (_key < 0)
//{
//printf("error in IVTree-insert: empty string\n");
//return false;
//}
//this->CopyToTransfer(_str, _len, 2);
//const Bstr* val = &(this->transfer[2]);
this->request = 0;
IVNode* ret;
if (this->root == NULL) //tree is empty
{
leaves_tail = leaves_head = root = new IVLeafNode;
request += IVNode::LEAF_SIZE;
this->height = 1;
root->setHeight(1); //add to heap later
}
//this->prepare(this->root); //root must be in-mem
if (root->getNum() == IVNode::MAX_KEY_NUM)
{
IVNode* father = new IVIntlNode;
request += IVNode::INTL_SIZE;
father->addChild(root, 0);
ret = root->split(father, 0);
if (ret->isLeaf() && ret->getNext() == NULL)
this->leaves_tail = ret;
if (ret->isLeaf())
request += IVNode::LEAF_SIZE;
else
request += IVNode::INTL_SIZE;
this->height++; //height rises only when root splits
//WARN: height area in Node: 4 bit!
father->setHeight(this->height); //add to heap later
this->TSM->updateHeap(ret, ret->getRank(), false);
this->root = father;
}
IVNode* p = this->root;
IVNode* q;
int i;
while (!p->isLeaf())
{
//j = p->getNum();
//for(i = 0; i < j; ++i)
//if(bstr < *(p->getKey(i)))
//break;
//NOTICE: using binary search is better here
i = p->searchKey_less(_key);
q = p->getChild(i);
this->prepare(q);
if (q->getNum() == IVNode::MAX_KEY_NUM)
{
ret = q->split(p, i);
if (ret->isLeaf() && ret->getNext() == NULL)
this->leaves_tail = ret;
if (ret->isLeaf())
request += IVNode::LEAF_SIZE;
else
request += IVNode::INTL_SIZE;
//BETTER: in loop may update multiple times
this->TSM->updateHeap(ret, ret->getRank(), false);
this->TSM->updateHeap(q, q->getRank(), true);
this->TSM->updateHeap(p, p->getRank(), true);
if (_key < p->getKey(i))
p = q;
else
p = ret;
}
else
{
p->setDirty();
this->TSM->updateHeap(p, p->getRank(), true);
p = q;
}
}
//j = p->getNum();
//for(i = 0; i < j; ++i)
//if(bstr < *(p->getKey(i)))
//break;
i = p->searchKey_less(_key);
//insert existing key is ok, but not inserted in
//however, the tree-shape may change due to possible split in former code
bool ifexist = false;
if (i > 0 && _key == p->getKey(i - 1))
ifexist = true;
else
{
p->addKey(_key, i);
p->addValue(this->value_list, i, _str, _len, true);
p->addNum();
//NOTICE: is this is a vlist, then it will be freed, and should not be included in the request memory
if(!VList::isLongList(_len))
{
request += _len;
}
//request += val->getLen();
p->setDirty();
this->TSM->updateHeap(p, p->getRank(), true);
//_key->clear();
//_value->clear();
}
this->TSM->request(request);
return !ifexist; //QUERY(which case:return false)
}
bool
IVTree::modify(unsigned _key, char* _str, unsigned _len)
{
//if (_key < 0)
//{
//printf("error in IVTree-modify: empty string\n");
//return false;
//}
//this->CopyToTransfer(_str, _len, 2); //not check value
//const Bstr* val = &(this->transfer[2]);
this->request = 0;
int store;
IVNode* ret = this->find(_key, &store, true);
if (ret == NULL || store == -1 || _key != ret->getKey(store)) //tree is empty or not found
{
cerr << "tree is empty or not found" << endl;
return false;
}
//cout<<"IVTree::modify() - key is found, now to remove"<<endl;
//NOTICE+DEBUG: if this value is a long list, then it is not saved in memory, here should return 0 in Bstr
unsigned len = ret->getValue(store)->getLen();
if(ret->getValue(store)->isBstrLongList())
{
len = 0;
}
ret->setValue(this->value_list, store, _str, _len, true);
//ret->setValue(val, store, true);
//cout<<"value reset"<<endl;
//cout<<"newlen: "<<val->getLen()<<" oldlen: "<<len<<endl;
//request += (val->getLen() - len);
if(!VList::isLongList(_len))
{
this->request += _len;
}
//this->request = val->getLen();
this->request -= len;
ret->setDirty();
//cout<<"to request"<<endl;
this->TSM->request(request);
//cout<<"memory requested"<<endl;
return true;
}
//this function is useful for search and modify, and range-query
IVNode* //return the first key's position that >= *_key
IVTree::find(unsigned _key, int* _store, bool ifmodify)
{ //to assign value for this->bstr, function shouldn't be const!
if (this->root == NULL)
return NULL; //IVTree Is Empty
IVNode* p = root;
int i, j;
while (!p->isLeaf())
{
if (ifmodify)
p->setDirty();
//j = p->getNum();
//for(i = 0; i < j; ++i) //BETTER(Binary-Search)
//if(bstr < *(p->getKey(i)))
//break;
i = p->searchKey_less(_key);
p = p->getChild(i);
this->prepare(p);
}
j = p->getNum();
//for(i = 0; i < j; ++i)
//if(bstr <= *(p->getKey(i)))
//break;
i = p->searchKey_lessEqual(_key);
if (i == j)
*_store = -1; //Not Found
else
*_store = i;
return p;
}
/*
Node*
IVTree::find(unsigned _len, const char* _str, int* store) const
{
}
*/
bool
IVTree::remove(unsigned _key)
{
//if (_key < 0)
//{
//printf("error in IVTree-remove: empty string\n");
//return false;
//}
this->request = 0;
IVNode* ret;
if (this->root == NULL) //tree is empty
return false;
IVNode* p = this->root;
IVNode* q;
int i, j;
while (!p->isLeaf())
{
j = p->getNum();
//for(i = 0; i < j; ++i)
//if(bstr < *(p->getKey(i)))
//break;
i = p->searchKey_less(_key);
q = p->getChild(i);
this->prepare(q);
if (q->getNum() < IVNode::MIN_CHILD_NUM) //==MIN_KEY_NUM
{
if (i > 0)
this->prepare(p->getChild(i - 1));
if (i < j)
this->prepare(p->getChild(i + 1));
ret = q->coalesce(p, i);
if (ret != NULL)
this->TSM->updateHeap(ret, 0, true);//non-sense node
this->TSM->updateHeap(q, q->getRank(), true);
if (q->isLeaf())
{
if (q->getPrev() == NULL)
this->leaves_head = q;
if (q->getNext() == NULL)
this->leaves_tail = q;
}
if (p->getNum() == 0) //root shrinks
{
//this->leaves_head = q;
this->root = q;
this->TSM->updateHeap(p, 0, true); //instead of delete p
this->height--;
}
}
else
p->setDirty();
this->TSM->updateHeap(p, p->getRank(), true);
p = q;
}
bool flag = false;
//j = p->getNum(); //LeafNode(maybe root)
//for(i = 0; i < j; ++i)
// if(bstr == *(p->getKey(i)))
// {
// request -= p->getKey(i)->getLen();
// request -= p->getValue(i)->getLen();
// p->subKey(i, true); //to release
// p->subValue(i, true); //to release
// p->subNum();
// if(p->getNum() == 0) //root leaf 0 key
// {
// this->root = NULL;
// this->leaves_head = NULL;
// this->leaves_tail = NULL;
// this->height = 0;
// this->TSM->updateHeap(p, 0, true); //instead of delete p
// }
// p->setDirty();
// flag = true;
// break;
// }
i = p->searchKey_equal(_key);
//WARN+NOTICE:here must check, because the key to remove maybe not exist
if (i != (int)p->getNum())
{
if(!p->getValue(i)->isBstrLongList())
{
request -= p->getValue(i)->getLen();
}
p->subKey(i); //to release
p->subValue(this->value_list, i, true); //to release
p->subNum();
if (p->getNum() == 0) //root leaf 0 key
{
this->root = NULL;
this->leaves_head = NULL;
this->leaves_tail = NULL;
this->height = 0;
this->TSM->updateHeap(p, 0, true); //instead of delete p
}
p->setDirty();
flag = true;
}
this->TSM->request(request);
return flag; //i == j, not found
}
const Bstr*
IVTree::getRangeValue()
{
if (this->stream == NULL)
{
fprintf(stderr, "IVTree::getRangeValue(): no results now!\n");
return NULL;
}
if (this->stream->isEnd())
{
fprintf(stderr, "IVTree::getRangeValue(): read till end now!\n");
return NULL;
}
//NOTICE:this is one record, and donot free the memory!
//NOTICE:Bstr[] but only one element, used as Bstr*
return this->stream->read();
}
void
IVTree::resetStream()
{
if (this->stream == NULL)
{
fprintf(stderr, "no results now!\n");
return;
}
this->stream->setEnd();
}
//TODO: change to using value list, getValue() maybe not get real long list
bool //special case: not exist, one-edge-case
IVTree::range_query(unsigned _key1, unsigned _key2)
{ //the range is: *_key1 <= x < *_key2
//if(_key1 <0 && _key2 <0)
//return false;
//ok to search one-edge, requiring only one be negative
//find and write value
int store1, store2;
IVNode *p1, *p2;
if (_key1 >= 0)
{
request = 0;
p1 = this->find(_key1, &store1, false);
if (p1 == NULL || store1 == -1)
return false; //no element
this->TSM->request(request);
}
else
{
p1 = this->leaves_head;
store1 = 0;
}
if (_key2 >= 0)
{ //QUERY: another strategy is to getnext and compare every time to tell end
request = 0;
p2 = this->find(_key2, &store2, false);
if (p2 == NULL)
return false;
else if (store2 == -1)
store2 = p2->getNum();
else if (store2 == 0)
{
p2 = p2->getPrev();
if (p2 == NULL)
return false; //no element
store2 = p2->getNum();
}
this->TSM->request(request);
}
else
{
p2 = this->leaves_tail;
store2 = p2->getNum();
}
IVNode* p = p1;
unsigned i, l, r;
//get the num of answers first, not need to prepare the node
unsigned ansNum = 0;
while (true)
{
//request = 0;
//this->prepare(p);
if (p == p1)
l = store1;
else
l = 0;
if (p == p2)
r = store2;
else
r = p->getNum();
ansNum += (r - l);
//this->TSM->request(request);
if (p != p2)
p = p->getNext();
else
break;
}
if (this->stream != NULL)
{
delete this->stream;
this->stream = NULL;
}
vector<unsigned> keys;
vector<bool> desc;
this->stream = new Stream(keys, desc, ansNum, 1, false);
p = p1;
while (1)
{
request = 0;
this->prepare(p);
if (p == p1)
l = store1;
else
l = 0;
if (p == p2)
r = store2;
else
r = p->getNum();
for (i = l; i < r; ++i)
{
//NOTICE:Bstr* in an array, used as Bstr[]
//DEBUG+TODO: if long list?? clean
this->stream->write(p->getValue(i));
}
this->TSM->request(request);
if (p != p2)
p = p->getNext();
else
break;
}
this->stream->setEnd();
return true;
}
bool
IVTree::save() //save the whole tree to disk
{
#ifdef DEBUG_KVSTORE
printf("now to save tree!\n");
#endif
if (TSM->writeTree(this->root))
return true;
else
return false;
}
void
IVTree::release(IVNode* _np) const
{
if (_np == NULL) return;
if (_np->isLeaf())
{
delete _np;
return;
}
int cnt = _np->getNum();
for (; cnt >= 0; --cnt)
release(_np->getChild(cnt));
delete _np;
}
void
IVTree::AddIntoCache(TYPE_PREDICATE_ID _id)
{
char* _tmp = NULL;
unsigned _len;
this->search(_id, _tmp, _len);
// cout << "len is " << len << endl;
this->value_list->AddIntoCache(_id, _tmp, _len);
delete [] _tmp;
}
void
IVTree::AddIntoCache(TYPE_ENTITY_LITERAL_ID _id)
{
char* _tmp = NULL;
unsigned _len;
this->search(_id, _tmp, _len);
// cout << "len is " << len << endl;
this->value_list->AddIntoCache(_id, _tmp, _len);
delete [] _tmp;
}
IVTree::~IVTree()
{
//cout << "IVTree" << endl;
//cout << "delete Vlist" << endl;
delete this->value_list;
value_list = NULL;
//cout << "delete stream" << endl;
delete this->stream;//maybe NULL
this->stream = NULL;
//cout << "delete TSM" << endl;
delete TSM;
TSM = NULL;
#ifdef DEBUG_KVSTORE
//printf("already empty the buffer, now to delete all nodes in tree!\n");
#endif
//recursively delete each Node
//cout << "release" << endl;
release(root);
//cout << "~IVTree done" << endl;
}
void
IVTree::print(string s)
{
#ifdef DEBUG_KVSTORE
fputs(Util::showtime().c_str(), Util::debug_kvstore);
fputs("Class IVTree\n", Util::debug_kvstore);
fputs("Message: ", Util::debug_kvstore);
fputs(s.c_str(), Util::debug_kvstore);
fputs("\n", Util::debug_kvstore);
fprintf(Util::debug_kvstore, "Height: %d\n", this->height);
if (s == "tree" || s == "TREE")
{
if (this->root == NULL)
{
fputs("Null IVTree\n", Util::debug_kvstore);
return;
}
IVNode** ns = new IVNode*[this->height];
int* ni = new int[this->height];
IVNode* np;
int i, pos = 0;
ns[pos] = this->root;
ni[pos] = this->root->getNum();
pos++;
while (pos > 0)
{
np = ns[pos - 1];
i = ni[pos - 1];
this->prepare(np);
if (np->isLeaf() || i < 0) //LeafNode or ready IntlNode
{ //child-num ranges: 0~num
if (s == "tree")
np->print("node");
else
np->print("NODE"); //print full node-information
pos--;
continue;
}
else
{
ns[pos] = np->getChild(i);
ni[pos - 1]--;
ni[pos] = ns[pos]->getNum();
pos++;
}
}
delete[] ns;
delete[] ni;
}
else if (s == "LEAVES" || s == "leaves")
{
IVNode* np;
for (np = this->leaves_head; np != NULL; np = np->getNext())
{
this->prepare(np);
if (s == "leaves")
np->print("node");
else
np->print("NODE");
}
}
else if (s == "check tree")
{
//check the tree, if satisfy B+ definition
//TODO
}
else;
#endif
}

101
KVstore/IVTree/IVTree.h Normal file
View File

@ -0,0 +1,101 @@
/*=============================================================================
# Filename: IVTree.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:44
# Description: ID2valueList, including s2po, p2so and o2ps
=============================================================================*/
#ifndef _KVSTORE_IVTREE_IVTREE_H
#define _KVSTORE_IVTREE_IVTREE_H
#include "../../Util/Util.h"
#include "../../Util/Stream.h"
#include "../../Util/VList.h"
#include "node/IVNode.h"
#include "node/IVIntlNode.h"
#include "node/IVLeafNode.h"
#include "storage/IVStorage.h"
//TODO: for long list, do not read in time, just on need
//the memory is kept with the node, updat ewith node
//NOTICE: to release the node, maybe the value list is NULL
//value bstr: unsigned=address, NULL
//BETTER?: build a new block store for long list??
//NOTICE: we do not need to use transfer bstr here, neithor for two directions
//when insert/query, we do not release the value in kvstore
class IVTree
{
protected:
unsigned height; //0 indicates an empty tree
IVNode* root;
IVNode* leaves_head; //the head of LeafNode-list
IVNode* leaves_tail; //the tail of LeafNode-list
std::string mode; //BETTER(to use enum)
IVStorage* TSM; //Tree-Storage-Manage
//BETTER:multiple stream maybe needed:)
Stream* stream;
//always alloc one more byte than length, then user can add a '\0'
//to get a real string, instead of new and copy
//other operations will be harmful to search, so store value in
//transfer temporally, while length adjusted.
//TODO: in multi-user case, multiple-search will cause problem,
//so lock is a must. Add lock to transfer is better than to add
//lock to every key/value. However, modify requires a lock for a
//key/value, and multiple search for different keys are ok!!!
//Bstr transfer;
//unsigned transfer_size;
//Bstr transfer[3]; //0:transfer value searched; 1:copy key-data from const char*; 2:copy val-data from const char*
//unsigned transfer_size[3];
//tree's operations should be atom(if read nodes)
//sum the request and send to IVStorage at last
//ensure that all nodes operated are in memory
long long request;
void prepare(IVNode* _np);
std::string storepath;
std::string filename; //ok for user to change
/* some private functions */
std::string getFilePath(); //in UNIX system
//void CopyToTransfer(const char* _str, unsigned _len, unsigned _index);
//void CopyToTransfer(const char* _str, unsigned _len);
void release(IVNode* _np) const;
//very long value list are stored in a separate file(with large block)
//
//NOTICE: according to the summary result, 90% value lists are just below 100 bytes
//<10%: 5000000~100M bytes
VList* value_list;
public:
IVTree(); //always need to initial transfer
IVTree(std::string _storepath, std::string _filename, std::string _mode, unsigned long long _buffer_size);
unsigned getHeight() const;
void setHeight(unsigned _h);
IVNode* getRoot() const;
//void setRoot(Node* _root);
//insert, search, remove, set
bool search(unsigned _key, char*& _str, unsigned& _len);
bool insert(unsigned _key, char* _str, unsigned _len);
bool modify(unsigned _key, char* _str, unsigned _len);
IVNode* find(unsigned _key, int* store, bool ifmodify);
bool remove(unsigned _key);
const Bstr* getRangeValue();
void resetStream();
bool range_query(unsigned _key1, unsigned _key2);
bool save();
void AddIntoCache(TYPE_PREDICATE_ID _id);
void AddIntoCache(TYPE_ENTITY_LITERAL_ID _id);
~IVTree();
void print(std::string s); //DEBUG(print the tree)
};
//NOTICE: need to save tree manually before delete, otherwise will cause problem.
//(problem range between two extremes: not-modified, totally-modified)
//After saved, it's ok to continue operations on tree!
#endif

188
KVstore/IVTree/heap/IVHeap.cpp Normal file
View File

@ -0,0 +1,188 @@
/*=============================================================================
# Filename: IVHeap.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:37
# Description: achieve functions in IVHeap.h
=============================================================================*/
#include "IVHeap.h"
using namespace std;
IVHeap::IVHeap()
{
this->length = this->size = 0;
this->heap = NULL;
}
IVHeap::IVHeap(unsigned _size)
{
this->length = 0;
this->size = _size;
this->heap = (IVNode**)malloc(this->size * sizeof(IVNode*)); //not use 4 or 8
//this->heap = new IVNode*[this->size];
if (this->heap == NULL)
{
this->print("error in IVHeap: Allocation fail!");
exit(1);
}
/*
this->npmap = (Map*)malloc(this->size * sizeof(struct Map));
if(this->npmap == NULL)
{
this->print("error in IVHeap: Allocation fail!");
exit(1);
}
*/
}
IVNode*
IVHeap::getTop() const
{
if (this->length > 0)
return this->heap[0];
else
return NULL;
}
unsigned
IVHeap::getLen() const
{
return this->length;
}
unsigned
IVHeap::getSize() const
{
return this->size;
}
bool
IVHeap::isEmpty() const
{
return this->length == 0;
}
bool
IVHeap::insert(IVNode* _np)
{
if (this->length == this->size) //when full, reallocate
{
cout<<"check: double the heap"<<endl;
this->heap = (IVNode**)realloc(this->heap, 2 * this->size * sizeof(IVNode*));
if (this->heap == NULL)
{
print("error in isert: Reallocation fail!");
return false;
}
/*
this->npmap = (struct Map*)realloc(this->npmap, 2 * this->size * sizeof(struct Map));
if(this->npmap == NULL)
{
print("error in insert: Reallocation fail!");
return false;
}
*/
this->size = 2 * this->size;
}
unsigned i = this->length, j;
while (i != 0)
{
j = (i - 1) / 2;
if (_np->getRank() >= this->heap[j]->getRank())
break;
heap[i] = heap[j];
//this->npmap[k].pos = i; //adjust the position
i = j;
}
this->heap[i] = _np;
this->length++;
return true;
}
bool
IVHeap::remove()
{
if (this->length == 0)
{
print("error in remove: remove from empty heap!");
return false;
}
//Node* tp = this->heap[0];
this->length--;
if (this->length == 0)
return true;
IVNode* xp = this->heap[this->length];
unsigned i = 0, j = 1;
while (j < this->length)
{
if (j < this->length - 1 && this->heap[j]->getRank() > this->heap[j + 1]->getRank())
j++;
if (xp->getRank() <= this->heap[j]->getRank())
break;
this->heap[i] = this->heap[j];
i = j;
j = 2 * i + 1;
}
this->heap[i] = xp;
return true;
}
bool
IVHeap::modify(IVNode* _np, bool _flag) //control direction
{
//search and adjust
unsigned i, j;
for (i = 0; i < this->length; ++i)
if (this->heap[i] == _np)
break;
if (_flag == true) //move up
{
while (i != 0)
{
j = (i - 1) / 2;
if (_np->getRank() < heap[j]->getRank())
{
heap[i] = heap[j];
heap[j] = _np;
i = j;
}
else
break;
}
}
else //move down
{
j = 2 * i + 1;
while (j < this->length)
{
if (j < this->length - 1 && heap[j]->getRank() > heap[j + 1]->getRank())
j++;
if (heap[j]->getRank() < _np->getRank())
{
heap[i] = heap[j];
heap[j] = _np;
i = j;
}
else
break;
}
}
return true;
}
IVHeap::~IVHeap()
{
//delete[] this->heap;
free(this->heap);
this->heap = NULL;
this->length = this->size = 0;
}
void
IVHeap::print(string s)
{
#ifdef DEBUG_KVSTORE
#endif
}

41
KVstore/IVTree/heap/IVHeap.h Normal file
View File

@ -0,0 +1,41 @@
/*=============================================================================
# Filename: IVHeap.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:37
# Description: set and deal of IVNode*s in memory
=============================================================================*/
#ifndef _KVSTORE_IVTREE_HEAP_IVHEAP_H
#define _KVSTORE_IVTREE_HEAP_IVHEAP_H
#include "../../../Util/Util.h"
#include "../node/IVNode.h"
/* add, sub, modify: all can be done within O(logn) using adjust-function */
//QUERY: when modified, finding right position consumes O(n). How about keeping smallest?
//(add O(1), sub O(2n), modify O(n)
//TODO: to solve this probem, use another hash: (pointer, pos), to find the right position of
//given p in O(lgn) time
class IVHeap
{
private:
IVNode** heap; //dynamic array
unsigned length; //valid elements num
unsigned size; //max-size of heap
public:
IVHeap();
IVHeap(unsigned _size);
IVNode* getTop() const; //return the top element
unsigned getLen() const;
unsigned getSize() const;
bool isEmpty() const;
bool insert(IVNode* _np); //insert and adjust
bool remove(); //remove top and adjust
bool modify(IVNode* _np, bool _flag); //searech modified element and adjust
~IVHeap();
void print(std::string s); //DEBUG
};
#endif

293
KVstore/IVTree/node/IVIntlNode.cpp Normal file
View File

@ -0,0 +1,293 @@
/*=============================================================================
# Filename: IVIntlNode.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:40
# Description: achieve functions in IVIntlNode.h
=============================================================================*/
#include "IVIntlNode.h"
using namespace std;
/*
void
IVIntlNode::AllocChilds()
{
childs = (Node**)malloc(sizeof(Node*) * MAX_CHILD_NUM);
}
*/
IVIntlNode::IVIntlNode()
{
memset(childs, 0, sizeof(IVNode*) * MAX_CHILD_NUM);
//this->AllocChilds();
}
IVIntlNode::IVIntlNode(bool isVirtual) //call father-class's constructor automaticlly
{
memset(childs, 0, sizeof(IVNode*) * MAX_CHILD_NUM);
//this->AllocChilds();
}
/*
IVIntlNode::IntlNode(Storage* TSM) //QUERY
{
TSM->readNode(this, Storage::OVER);
}
*/
void
IVIntlNode::Virtual()
{
//this->FreeKeys();
this->release();
this->delMem();
}
void
IVIntlNode::Normal()
{
this->AllocKeys();
this->setMem();
}
IVNode*
IVIntlNode::getChild(int _index) const
{
int num = this->getNum();
if (_index < 0 || _index > num) //num keys, num+1 childs
{
//print(string("error in getChild: Invalid index ") + Util::int2string(_index));
return NULL;
}
else
return childs[_index];
}
bool
IVIntlNode::setChild(IVNode* _child, int _index)
{
int num = this->getNum();
if (_index < 0 || _index > num)
{
print(string("error in setChild: Invalid index ") + Util::int2string(_index));
return false;
}
this->childs[_index] = _child;
return true;
}
bool
IVIntlNode::addChild(IVNode* _child, int _index)
{
int num = this->getNum();
if (_index < 0 || _index > num + 1)
{
print(string("error in addChild: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
for (i = num; i >= _index; --i) //DEBUG: right bounder!!!
childs[i + 1] = childs[i];
childs[_index] = _child;
return true;
}
bool
IVIntlNode::subChild(int _index)
{
int num = this->getNum();
if (_index < 0 || _index > num)
{
print(string("error in subchild: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
for (i = _index; i < num; ++i) //DEBUG: right bounder!!!
childs[i] = childs[i + 1];
return true;
}
unsigned
IVIntlNode::getSize() const
{
//unsigned sum = INTL_SIZE, num = this->getNum(), i;
//return sum;
return INTL_SIZE;
}
IVNode*
IVIntlNode::split(IVNode* _father, int _index)
{
int num = this->getNum();
IVNode* p = new IVIntlNode; //right child
p->setHeight(this->getHeight());
int i, k;
for (i = MIN_CHILD_NUM, k = 0; i < num; ++i, ++k)
{
p->addKey(this->keys[i], k);
p->addChild(this->childs[i], k);
p->addNum();
}
p->addChild(this->childs[i], k);
int tp = this->keys[MIN_KEY_NUM];
this->setNum(MIN_KEY_NUM);
_father->addKey(tp, _index);
_father->addChild(p, _index + 1); //DEBUG(check the index)
_father->addNum();
_father->setDirty();
p->setDirty();
this->setDirty();
return p;
}
IVNode*
IVIntlNode::coalesce(IVNode* _father, int _index)
{
//int num = this->getNum();
int i, j = _father->getNum(), k; //BETTER: unsigned?
IVNode* p;
int ccase = 0;
//const Bstr* bstr;
if (_index < j) //the right neighbor
{
p = _father->getChild(_index + 1);
k = p->getNum();
if ((unsigned)k > MIN_KEY_NUM)
ccase = 2;
else //==MIN_KEY_NUM
ccase = 1;
}
if (_index > 0) //the left neighbor
{
IVNode* tp = _father->getChild(_index - 1);
unsigned tk = tp->getNum();
if (ccase < 2)
{
if (ccase == 0)
ccase = 3;
if (tk > MIN_KEY_NUM)
ccase = 4;
}
if (ccase > 2)
{
p = tp;
k = tk;
}
}
unsigned tmp = 0;
switch (ccase)
{
case 1: //union right to this
this->addKey(_father->getKey(_index), this->getNum());
this->addNum();
for (i = 0; i < k; ++i)
{
this->addKey(p->getKey(i), this->getNum());
this->addChild(p->getChild(i), this->getNum());
this->addNum();
}
this->setChild(p->getChild(i), this->getNum());
_father->subKey(_index);
_father->subChild(_index + 1);
_father->subNum();
p->setNum(0);
//delete p;
break;
case 2: //move one form right
this->addKey(_father->getKey(_index), this->getNum());
_father->setKey(p->getKey(0), _index);
p->subKey(0);
this->addChild(p->getChild(0), this->getNum() + 1);
p->subChild(0);
this->addNum();
p->subNum();
break;
case 3: //union left to this
this->addKey(_father->getKey(_index - 1), 0);
this->addNum();
for (i = k; i > 0; --i)
{
int t = i - 1;
this->addKey(p->getKey(t), 0);
this->addChild(p->getChild(i), 0);
this->addNum();
}
this->addChild(p->getChild(0), 0);
_father->subKey(_index - 1);
_father->subChild(_index - 1);
_father->subNum();
p->setNum(0);
//delete p;
break;
case 4: //move one from left
tmp = p->getKey(k - 1);
p->subKey(k - 1);
this->addKey(_father->getKey(_index - 1), 0);
_father->setKey(tmp, _index - 1);
this->addChild(p->getChild(k), 0);
p->subChild(k);
this->addNum();
p->subNum();
break;
default:
print("error in coalesce: Invalid case!");
//printf("error in coalesce: Invalid case!");
}
_father->setDirty();
p->setDirty();
this->setDirty();
if (ccase == 1 || ccase == 3)
return p;
else
return NULL;
}
void
IVIntlNode::release()
{
if (!this->inMem())
return;
//unsigned num = this->getNum();
delete[] keys; //this will release all!!!
}
IVIntlNode::~IVIntlNode()
{
release();
//free(childs);
}
void
IVIntlNode::print(string s)
{
#ifdef DEBUG_KVSTORE
int num = this->getNum();
fputs(Util::showtime().c_str(), Util::debug_kvstore);
fputs("Class IVIntlNode\n", Util::debug_kvstore);
fputs("Message: ", Util::debug_kvstore);
fputs(s.c_str(), Util::debug_kvstore);
fputs("\n", Util::debug_kvstore);
if (s == "node" || s == "NODE")
{
fprintf(Util::debug_kvstore, "store: %u\tnum: %u\tflag: %u\n", this->store, num, this->flag);
/*
int i;
for (i = 0; i < num; ++i)
{
if (s == "node")
this->keys[i].print("bstr");
else
this->keys[i].print("BSTR");
}
*/
}
else if (s == "check node")
{
//TODO(check node, if satisfy B+ definition)
}
else;
#endif
}

48
KVstore/IVTree/node/IVIntlNode.h Normal file
View File

@ -0,0 +1,48 @@
/*=============================================================================
# Filename: IVIntlNode.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:40
# Description: the internal-node of a B+ tree
=============================================================================*/
#ifndef _KVSTORE_IVTREE_NODE_IVINTLNODE_H
#define _KVSTORE_IVTREE_NODE_IVINTLNODE_H
#include "IVNode.h"
class IVIntlNode : public IVNode
{
protected:
IVNode* childs[MAX_CHILD_NUM + 1];
//Node** childs;
//void AllocChilds();
public:
IVIntlNode();
IVIntlNode(bool isVirtual);
//IntlNode(Storage* TSM);
void Virtual();
void Normal();
IVNode* getChild(int _index) const;
bool setChild(IVNode* _child, int _index);
bool addChild(IVNode* _child, int _index);
bool subChild(int _index);
unsigned getSize() const;
IVNode* split(IVNode* _father, int _index);
IVNode* coalesce(IVNode* _father, int _index);
void release();
~IVIntlNode();
void print(std::string s); //DEBUG
/*non-sense functions: polymorphic
Node* getPrev() const;
Node* getNext() const;
const Bstr* getValue(int _index) const;
bool setValue(const Bstr* _value, int _index);
bool addValue(const Bstr* _value, int _index);
bool subValue(int _index);
void setPrev(Node* _prev);
void setNext(Node* _next);
*/
};
#endif

538
KVstore/IVTree/node/IVLeafNode.cpp Normal file
View File

@ -0,0 +1,538 @@
/*=============================================================================
# Filename: IVLeafNode.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:40
# Description: ahieve functions in IVLeafNode.h
=============================================================================*/
#include "IVLeafNode.h"
using namespace std;
void
IVLeafNode::AllocValues()
{
values = new Bstr[MAX_KEY_NUM];
}
/*
void
IVLeafNode::FreeValues()
{
delete[] values;
}
*/
IVLeafNode::IVLeafNode()
{
flag |= NF_IL; //leaf flag
prev = next = NULL;
AllocValues();
}
IVLeafNode::IVLeafNode(bool isVirtual)
{
flag |= NF_IL;
prev = next = NULL;
if (!isVirtual)
AllocValues();
}
/*
IVLeafNode::LeafNode(Storage* TSM)
{
AllocValues();
TSM->readNode(this, Storage::OVER);
}
*/
void
IVLeafNode::Virtual()
{
//this->FreeKeys();
//this->FreeValues();
this->release();
this->delMem();
}
void
IVLeafNode::Normal()
{
this->AllocKeys();
this->AllocValues();
this->setMem();
}
IVNode*
IVLeafNode::getPrev() const
{
return prev;
}
IVNode*
IVLeafNode::getNext() const
{
return next;
}
const Bstr*
IVLeafNode::getValue(int _index) const
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
//print(string("error in getValue: Invalid index ") + Util::int2string(_index));
return NULL;
}
else
return this->values + _index;
}
bool
IVLeafNode::setValue(const Bstr* _value, int _index, bool _ifcopy)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
print(string("error in setValue: Invalid index ") + Util::int2string(_index));
return false;
}
this->values[_index].release(); //NOTICE: only used in modify
if(_ifcopy)
{
this->values[_index].copy(_value);
}
else
{
this->values[_index] = *_value;
}
return true;
}
bool
IVLeafNode::getValue(VList* _vlist, int _index, char*& _str, unsigned& _len) const
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
//print(string("error in getValue: Invalid index ") + Util::int2string(_index));
return NULL;
}
//read long list
if(this->values[_index].isBstrLongList())
{
#ifdef DEBUG_VLIST
cout<<"this is a vlist in get()"<<endl;
#endif
unsigned block_num = this->values[_index].getLen();
_vlist->readValue(block_num, _str, _len, this->keys[_index]);
}
else
{
_str = this->values[_index].getStr();
_len = this->values[_index].getLen();
}
return true;
}
bool
IVLeafNode::setValue(VList* _vlist, int _index, char* _str, unsigned _len, bool ifcopy)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
print(string("error in setValue: Invalid index ") + Util::int2string(_index));
return false;
}
if(this->values[_index].isBstrLongList())
{
#ifdef DEBUG_VLIST
cout<<"this is a vlist in set()"<<endl;
#endif
unsigned block_num = this->values[_index].getLen();
_vlist->removeValue(block_num);
}
else
{
this->values[_index].release(); //NOTICE: only used in modify
}
//DEBUG: we do not need to copy here
//we just need to ensure that the pointer's memory is not released
//if (ifcopy)
//{
//this->values[_index].copy(_value);
//}
//else
//{
//this->values[_index] = *_value;
if(VList::isLongList(_len))
{
unsigned block_num = _vlist->writeValue(_str, _len);
this->values[_index].setStr(NULL);
this->values[_index].setLen(block_num);
//NOTICE: we need to free the long list value
delete[] _str;
}
else
{
this->values[_index].setStr(_str);
this->values[_index].setLen(_len);
}
//}
return true;
}
bool
IVLeafNode::addValue(VList* _vlist, int _index, char* _str, unsigned _len, bool ifcopy)
{
int num = this->getNum();
if (_index < 0 || _index > num)
{
print(string("error in addValue: Invalid index ") + Util::int2string(_index));
return false;
}
for (int i = num - 1; i >= _index; --i)
this->values[i + 1] = this->values[i];
//if (ifcopy)
//this->values[_index].copy(_value);
//else
//this->values[_index] = *_value;
if(VList::isLongList(_len))
{
#ifdef DEBUG_VLIST
cout<<"this is a vlist in add()"<<endl;
#endif
unsigned block_num = _vlist->writeValue(_str, _len);
this->values[_index].setStr(NULL);
this->values[_index].setLen(block_num);
//NOTICE: we need to free the long list value
delete[] _str;
#ifdef DEBUG_VLIST
//cout<<"to check vlist: "<<this->values[_index].getLen()<<endl;
#endif
}
else
{
this->values[_index].setStr(_str);
this->values[_index].setLen(_len);
}
//this->values[_index].setStr(_str);
//this->values[_index].setLen(_len);
return true;
}
bool
IVLeafNode::subValue(VList* _vlist, int _index, bool ifdel)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
print(string("error in subValue: Invalid index ") + Util::int2string(_index));
return false;
}
if(this->values[_index].isBstrLongList())
{
unsigned block_num = this->values[_index].getLen();
_vlist->removeValue(block_num);
}
else
{
if (ifdel)
{
values[_index].release();
}
}
for (int i = _index; i < num - 1; ++i)
this->values[i] = this->values[i + 1];
return true;
}
bool
IVLeafNode::addValue(const Bstr* _value, int _index, bool ifcopy)
{
int num = this->getNum();
if (_index < 0 || _index > num)
{
print(string("error in addValue: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
for (i = num - 1; i >= _index; --i)
this->values[i + 1] = this->values[i];
if (ifcopy)
this->values[_index].copy(_value);
else
this->values[_index] = *_value;
return true;
}
bool
IVLeafNode::subValue(int _index, bool ifdel)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
print(string("error in subValue: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
if (ifdel)
values[_index].release();
for (i = _index; i < num - 1; ++i)
this->values[i] = this->values[i + 1];
return true;
}
void
IVLeafNode::setPrev(IVNode* _prev)
{
this->prev = _prev;
}
void
IVLeafNode::setNext(IVNode* _next)
{
this->next = _next;
}
unsigned
IVLeafNode::getSize() const
{
unsigned sum = LEAF_SIZE, num = this->getNum(), i;
for (i = 0; i < num; ++i)
{
sum += values[i].getLen();
}
return sum;
}
IVNode*
IVLeafNode::split(IVNode* _father, int _index)
{
int num = this->getNum();
IVNode* p = new IVLeafNode; //right child
p->setHeight(this->getHeight()); //NOTICE: assign height for new node
p->setNext(this->next);
this->setNext(p);
p->setPrev(this);
int i, k;
for (i = MIN_KEY_NUM, k = 0; i < num; ++i, ++k)
{
p->addKey(this->keys[i], k);
p->addValue(this->values + i, k);
p->addNum();
}
unsigned tp = this->keys[MIN_KEY_NUM];
this->setNum(MIN_KEY_NUM);
_father->addKey(tp, _index);
_father->addChild(p, _index + 1); //DEBUG(check the index)
_father->addNum();
_father->setDirty();
p->setDirty();
this->setDirty();
return p;
}
IVNode*
IVLeafNode::coalesce(IVNode* _father, int _index)
{ //add a key or coalesce a neighbor to this
int i, j = _father->getNum(), k; //BETTER: unsigned?
IVNode* p = NULL;
int ccase = 0;
//const Bstr* bstr;
if (_index < j) //the right neighbor
{
p = _father->getChild(_index + 1);
k = p->getNum();
if ((unsigned)k > MIN_KEY_NUM)
ccase = 2;
else //==MIN_KEY_NUM
ccase = 1;
}
if (_index > 0) //the left neighbor
{
IVNode* tp = _father->getChild(_index - 1);
unsigned tk = tp->getNum();
if (ccase < 2)
{
if (ccase == 0)
ccase = 3;
if (tk > MIN_KEY_NUM)
ccase = 4;
}
if (ccase > 2)
{
p = tp;
k = tk;
}
}
int tmp = 0;
switch (ccase)
{
case 1: //union right to this
for (i = 0; i < k; ++i)
{
this->addKey(p->getKey(i), this->getNum());
this->addValue(p->getValue(i), this->getNum());
this->addNum();
}
_father->subKey(_index);
_father->subChild(_index + 1);
_father->subNum();
this->next = p->getNext();
if (this->next != NULL)
this->next->setPrev(this);
p->setNum(0); //NOTICE: adjust num before delete!
//delete p;
break;
case 2: //move one from right
this->addKey(p->getKey(0), this->getNum());
_father->setKey(p->getKey(1), _index);
p->subKey(0);
this->addValue(p->getValue(0), this->getNum());
p->subValue(0);
this->addNum();
p->subNum();
break;
case 3: //union left to this
//BETTER: move all keys/etc one time
for (i = k; i > 0; --i)
{
int t = i - 1;
this->addKey(p->getKey(t), 0);
this->addValue(p->getValue(t), 0);
this->addNum();
}
_father->subKey(_index - 1);
_father->subChild(_index - 1);
_father->subNum();
this->prev = p->getPrev();
if (this->prev != NULL) //else: leaves-list
this->prev->setNext(this);
p->setNum(0);
//delete p;
break;
case 4: //move one from left
tmp = p->getKey(k - 1);
p->subKey(k - 1);
this->addKey(tmp, 0);
_father->setKey(tmp, _index - 1);
this->addValue(p->getValue(k - 1), 0);
p->subValue(k - 1);
this->addNum();
p->subNum();
break;
default:
print("error in coalesce: Invalid case!");
//printf("error in coalesce: Invalid case!");
}
_father->setDirty();
p->setDirty();
this->setDirty();
if (ccase == 1 || ccase == 3)
return p;
else
return NULL;
}
void
IVLeafNode::release()
{
if (!this->inMem())
return;
unsigned num = this->getNum();
/*
for(int i = 0; i < num; ++i)
{
keys[i].release();
values[i].release();
}
*/
for (unsigned i = num; i < MAX_KEY_NUM; ++i)
{
values[i].clear();
}
delete[] keys;
delete[] values;
}
IVLeafNode::~IVLeafNode()
{
release();
}
void
IVLeafNode::print(string s)
{
#ifdef DEBUG_KVSTORE
unsigned num = this->getNum();
fputs(Util::showtime().c_str(), Util::debug_kvstore);
fputs("Class IVLeafNode\n", Util::debug_kvstore);
fputs("Message: ", Util::debug_kvstore);
fputs(s.c_str(), Util::debug_kvstore);
fputs("\n", Util::debug_kvstore);
unsigned i;
if (s == "NODE")
{
fprintf(Util::debug_kvstore, "store: %u\tnum: %u\tflag: %u\n", this->store, num, this->flag);
fprintf(Util::debug_kvstore, "prev: %p\tnext: %p\n", this->prev, this->next);
for (i = 0; i < num; ++i)
{
//this->keys[i].print("BSTR");
this->values[i].print("BSTR");
}
}
else if (s == "node")
{
fprintf(Util::debug_kvstore, "store: %u\tnum: %u\tflag: %u\n", this->store, num, this->flag);
fprintf(Util::debug_kvstore, "prev: %p\tnext: %p\n", this->prev, this->next);
}
else if (s == "check node")
{
//check the node, if satisfy B+ definition
bool flag = true;
if (num < MIN_KEY_NUM || num > MAX_KEY_NUM)
flag = false;
if (flag)
{
for (i = 1; i < num; ++i)
{
if (keys[i] > keys[i - 1])
continue;
else
break;
}
if (i < num)
flag = false;
}
this->print("node");
if (flag)
fprintf(Util::debug_kvstore, "This node is good\n");
else
fprintf(Util::debug_kvstore, "This node is bad\n");
}
else;
#endif
}

58
KVstore/IVTree/node/IVLeafNode.h Normal file
View File

@ -0,0 +1,58 @@
/*=============================================================================
# Filename: IVLeafNode.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:39
# Description: the leaf-node of a B+ tree
=============================================================================*/
#ifndef _KVSTORE_IVTREE_NODE_IVLEAFNODE_H
#define _KVSTORE_IVTREE_NODE_IVLEAFNODE_H
#include "IVNode.h"
class IVLeafNode : public IVNode
{
protected:
IVNode* prev; //LeafNode
IVNode* next;
Bstr* values;
void AllocValues();
//void FreeValues();
public:
IVLeafNode();
IVLeafNode(bool isVirtual);
//LeafNode(Storage* TSM);
void Virtual();
void Normal();
IVNode* getPrev() const;
IVNode* getNext() const;
const Bstr* getValue(int _index) const;
bool setValue(const Bstr* _value, int _index, bool _ifcopy=false);
bool getValue(VList* _vlist, int _index, char*& _str, unsigned& _len) const;
bool setValue(VList* _vlist, int _index, char* _str, unsigned _len, bool ifcopy = false);
bool addValue(VList* _vlist, int _index, char* _str, unsigned _len, bool ifcopy = false);
bool subValue(VList* _vlist, int _index, bool ifdel = false);
bool addValue(const Bstr* _val, int _index, bool ifcopy = false);
bool subValue(int _index, bool ifdel = false);
void setPrev(IVNode* _prev);
void setNext(IVNode* _next);
unsigned getSize() const;
IVNode* split(IVNode* _father, int _index);
IVNode* coalesce(IVNode* _father, int _index);
void release();
~IVLeafNode();
void print(std::string s); //DEBUG
/*non-sense virtual function
Node* getChild(int _index) const;
bool addChild(Node* _child, int _index);
bool subChild(int _index);
*/
};
//BETTER: prev isn't a must, and reverse-range can be achieved using recursive-next
#endif

320
KVstore/IVTree/node/IVNode.cpp Normal file
View File

@ -0,0 +1,320 @@
/*=============================================================================
# Filename: IVNode.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:39
# Description: achieve functions in IVNode.h
=============================================================================*/
#include "IVNode.h"
using namespace std;
void
IVNode::AllocKeys()
{
keys = new unsigned[MAX_KEY_NUM];
}
/*
void
IVNode::FreeKeys()
{
delete[] keys;
}
*/
IVNode::IVNode()
{
store = flag = 0;
flag |= NF_IM;
AllocKeys();
}
IVNode::IVNode(bool isVirtual)
{
store = flag = 0;
if (!isVirtual)
{
flag |= NF_IM;
AllocKeys();
}
}
/*
IVNode::Node(Storage* TSM)
{
AllocKeys();
TSM->readIVNode(this, Storage::OVER);
}
*/
bool
IVNode::isLeaf() const
{
return this->flag & NF_IL;
}
bool
IVNode::isDirty() const
{
return this->flag & NF_ID;
}
void
IVNode::setDirty()
{
this->flag |= NF_ID;
}
void
IVNode::delDirty()
{
this->flag &= ~NF_ID;
}
bool
IVNode::inMem() const
{
return this->flag & NF_IM;
}
void
IVNode::setMem()
{
this->flag |= NF_IM;
}
void
IVNode::delMem()
{
this->flag &= ~NF_IM;
}
/*
bool
IVNode::isVirtual() const
{
return this->flag & NF_IV;
}
void
IVNode::setVirtual()
{
this->flag |= NF_IV;
}
void
IVNode::delVirtual()
{
this->flag &= ~NF_IV;
}
*/
unsigned
IVNode::getRank() const
{
return this->flag & NF_RK;
}
void
IVNode::setRank(unsigned _rank)
{
this->flag &= ~NF_RK;
this->flag |= _rank;
}
unsigned
IVNode::getHeight() const
{
return (this->flag & NF_HT) >> 20;
}
void
IVNode::setHeight(unsigned _h)
{
this->flag &= ~NF_HT;
this->flag |= (_h << 20);
}
unsigned
IVNode::getNum() const
{
return (this->flag & NF_KN) >> 12;
}
bool
IVNode::setNum(int _num)
{
if (_num < 0 || (unsigned)_num > MAX_KEY_NUM)
{
print(string("error in setNum: Invalid num ") + Util::int2string(_num));
return false;
}
this->flag &= ~NF_KN;
this->flag |= (_num << 12);
return true;
}
bool
IVNode::addNum()
{
if (this->getNum() + 1 > MAX_KEY_NUM)
{
print("error in addNum: Invalid!");
return false;
}
this->flag += (1 << 12);
return true;
}
bool
IVNode::subNum()
{
if (this->getNum() < 1)
{
print("error in subNum: Invalid!");
return false;
}
this->flag -= (1 << 12);
return true;
}
unsigned
IVNode::getStore() const
{
return this->store;
}
void
IVNode::setStore(unsigned _store)
{
this->store = _store;
}
unsigned
IVNode::getFlag() const
{
return flag;
}
void
IVNode::setFlag(unsigned _flag)
{
this->flag = _flag;
}
unsigned
IVNode::getKey(int _index) const
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
//print(string("error in getKey: Invalid index ") + Util::int2string(_index));
printf("error in getKey: Invalid index\n");
return -1;
}
else
return this->keys[_index];
}
bool
IVNode::setKey(unsigned _key, int _index)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
print(string("error in setKey: Invalid index ") + Util::int2string(_index));
return false;
}
keys[_index] = _key;
return true;
}
bool
IVNode::addKey(unsigned _key, int _index)
{
int num = this->getNum();
if (_index < 0 || _index > num)
{
print(string("error in addKey: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
//NOTICE: if num == MAX_KEY_NUM, will visit keys[MAX_KEY_NUM], not legal!!!
//however. tree operations ensure that: when node is full, not add but split first!
for (i = num - 1; i >= _index; --i)
keys[i + 1] = keys[i];
keys[_index] = _key;
return true;
}
bool
IVNode::subKey(int _index)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
{
print(string("error in subKey: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
for (i = _index; i < num - 1; ++i)
keys[i] = keys[i + 1];
return true;
}
int
IVNode::searchKey_less(unsigned _key) const
{
int num = this->getNum();
//for(i = 0; i < num; ++i)
//if(bstr < *(p->getKey(i)))
//break;
int low = 0, high = num - 1, mid = -1;
while (low <= high)
{
mid = (low + high) / 2;
if (this->keys[mid] > _key)
{
if (low == mid)
break;
high = mid;
}
else
{
low = mid + 1;
}
}
return low;
}
int
IVNode::searchKey_equal(unsigned _key) const
{
int num = this->getNum();
//for(i = 0; i < num; ++i)
// if(bstr == *(p->getKey(i)))
// {
int ret = this->searchKey_less(_key);
if (ret > 0 && this->keys[ret - 1] == _key)
return ret - 1;
else
return num;
}
int
IVNode::searchKey_lessEqual(unsigned _key) const
{
//int num = this->getNum();
//for(i = 0; i < num; ++i)
//if(bstr <= *(p->getKey(i)))
//break;
int ret = this->searchKey_less(_key);
if (ret > 0 && this->keys[ret - 1] == _key)
return ret - 1;
else
return ret;
}

123
KVstore/IVTree/node/IVNode.h Normal file
View File

@ -0,0 +1,123 @@
/*=============================================================================
# Filename: IVNode.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:38
# Description: basic Node class, father of IVIntlNode and IVLeafNode
=============================================================================*/
#ifndef _KVSTORE_IVTREE_NODE_IVNODE_H
#define _KVSTORE_IVTREE_NODE_IVNODE_H
#include "../../../Util/Util.h"
#include "../../../Util/Bstr.h"
#include "../../../Util/VList.h"
class IVNode //abstract basic class
{
public:
static const unsigned DEGREE = 2 * 63; //the degree of B+ tree
static const unsigned MAX_CHILD_NUM = DEGREE;
static const unsigned MIN_CHILD_NUM = DEGREE >> 1;
static const unsigned MAX_KEY_NUM = MAX_CHILD_NUM - 1; //max key-num
static const unsigned MIN_KEY_NUM = MIN_CHILD_NUM - 1; //min key-num
/* diffrent flags for tree-nodes, 32-bit put rank in low-bits means no need to move*/
static const unsigned NF_IL = 0x80000000; //is leaf
static const unsigned NF_ID = 0x00080000; //is dirty, in rank-area
static const unsigned NF_IM = 0x20000000; //in memory, not virtual
//static const unsigned NF_IV = 0x10000000; //is virtual
static const unsigned NF_RK = 0x00ffffff; //select-rank, in Storage
static const unsigned NF_HT = 0xf00000; //height area in rank
static const unsigned NF_KN = 0x07f000; //NOTICE: decided by DEGREE
static const unsigned INTL_SIZE = sizeof(int) * MAX_KEY_NUM;
static const unsigned LEAF_SIZE = INTL_SIZE + sizeof(Bstr) * MAX_KEY_NUM;
protected:
unsigned store; //store address, the BLock index
unsigned flag; //NF_RK, NF_IL,NF_ID, NF_IV, propety
//int num; //totle keys num
//Node* father; //point to father-node, which must be IntlNode
unsigned* keys;
void AllocKeys();
//void FreeKeys();
public:
IVNode();
IVNode(bool isVirtual);
bool isLeaf() const;
bool isDirty() const;
void setDirty();
void delDirty();
bool inMem() const;
void setMem();
void delMem();
//bool isVirtual() const;
//void setVirtual();
//void delVirtual();
unsigned getRank() const;
void setRank(unsigned _rank);
unsigned getHeight() const;
void setHeight(unsigned _h);
unsigned getNum() const;
bool setNum(int _num);
bool addNum();
bool subNum();
unsigned getStore() const;
void setStore(unsigned _store);
unsigned getFlag() const;
void setFlag(unsigned _flag);
unsigned getKey(int _index) const; //need to check the index
bool setKey(unsigned _key, int _index);
bool addKey(unsigned _key, int _index);
bool subKey(int _index);
//several binary key search utilities
int searchKey_less(unsigned _key) const;
int searchKey_equal(unsigned _key) const;
int searchKey_lessEqual(unsigned _key) const;
//virtual functions: polymorphic
virtual IVNode* getChild(int _index) const { return NULL; };
virtual bool setChild(IVNode* _child, int _index) { return true; };
virtual bool addChild(IVNode* _child, int _index) { return true; };
virtual bool subChild(int _index) { return true; };
virtual IVNode* getPrev() const { return NULL; };
virtual IVNode* getNext() const { return NULL; };
virtual const Bstr* getValue(int _index) const { return NULL; };
virtual bool setValue(const Bstr* _value, int _index, bool _ifcopy=false) { return true; };
virtual bool getValue(VList* _vlist, int _index, char*& _str, unsigned& _len) const { return NULL; };
virtual bool setValue(VList* _vlist, int _index, char* _str, unsigned _len, bool ifcopy = false) { return true; };
virtual bool addValue(VList* _vlist, int _index, char* _str, unsigned _len, bool ifcopy = false) { return true; };
virtual bool subValue(VList* _vlist, int _index, bool ifdel = false) { return true; };
virtual bool addValue(const Bstr* _val, int _index, bool ifcopy = false) { return true; };
virtual bool subValue(int _index, bool ifdel = false) { return true; };
virtual void setPrev(IVNode* _prev) {};
virtual void setNext(IVNode* _next) {};
//pure virtual functions
virtual void Virtual() = 0;
virtual void Normal() = 0;
virtual unsigned getSize() const = 0; //return all memory owned
virtual IVNode* split(IVNode* _father, int _index) = 0;
virtual IVNode* coalesce(IVNode* _father, int _index) = 0;
virtual void release() = 0; //release the node, only remain necessary information
virtual ~IVNode() {};
virtual void print(std::string s) = 0; //DEBUG(print the Node)
};
/*NOTICE(operations in release())
*To save memory, we can only remain store and flag(childs added for Leaf).
*However, in this way childs'pointers is ok to change, use Node** or Node*& is also nonsense
*because the pointer variable may die.
*Another way is only to release dynamic memory, and store thw whole, read the Bstr only to
*build. In this way nodes'pointer doesn't change, and operation is simplified, while memory
*is consumed a bit more. Because Bstrs consume the most memory, and memory-disk swapping is
*the most time-consuming thing, it seems to be a better way.
*WARN:when a node is in memory and not deleted, its basic content is always being! If nodes are
*really too many, this will cause disaster because we can't swap them out until tree is closed!
*To solve this problem, there should be two types of release-function: one to release Bstr, one
*to release the whole(pointer is invalid and rebuild problem)
*/
#endif

742
KVstore/IVTree/storage/IVStorage.cpp Normal file
View File

@ -0,0 +1,742 @@
/*=============================================================================
# Filename: IVStorage.cpp
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:43
# Description: achieve functions in IVStorage.h
=============================================================================*/
#include "IVStorage.h"
using namespace std;
IVStorage::IVStorage()
{ //not use ../logs/, notice the location of program
cur_block_num = SET_BLOCK_NUM;
filepath = "";
freelist = NULL;
treefp = NULL;
max_buffer_size = Util::MAX_BUFFER_SIZE;
heap_size = max_buffer_size / IVNode::INTL_SIZE;
freemem = max_buffer_size;
minheap = NULL;
this->value_list = NULL;
}
IVStorage::IVStorage(string& _filepath, string& _mode, unsigned* _height, unsigned long long _buffer_size, VList* _vlist)
{
cur_block_num = SET_BLOCK_NUM; //initialize
this->filepath = _filepath;
if (_mode == string("build"))
treefp = fopen(_filepath.c_str(), "w+b");
else if (_mode == string("open"))
treefp = fopen(_filepath.c_str(), "r+b");
else
{
print(string("error in IVStorage: Invalid mode ") + _mode);
return;
}
if (treefp == NULL)
{
print(string("error in IVStorage: Open error ") + _filepath);
return;
}
this->treeheight = _height; //originally set to 0
this->max_buffer_size = _buffer_size;
this->heap_size = this->max_buffer_size / IVNode::INTL_SIZE;
this->freemem = this->max_buffer_size;
this->freelist = new BlockInfo; //null-head
unsigned i, j, k; //j = (SuperNum-1)*BLOCK_SIZE
BlockInfo* bp;
if (_mode == "build")
{ //write basic information
i = 0;
fwrite(&i, sizeof(unsigned), 1, this->treefp); //height
fwrite(&i, sizeof(unsigned), 1, this->treefp); //rootnum
fwrite(&cur_block_num, sizeof(unsigned), 1, this->treefp); //current block num
fseek(this->treefp, BLOCK_SIZE, SEEK_SET);
bp = this->freelist;
j = cur_block_num / 8;
for (i = 0; i < j; ++i)
{
fputc(0, this->treefp);
for (k = 0; k < 8; ++k)
{
bp->next = new BlockInfo(i * 8 + k + 1, NULL);
bp = bp->next;
}
}
}
else //_mode == "open"
{
//read basic information
unsigned rootnum;
char c;
fread(this->treeheight, sizeof(unsigned), 1, this->treefp);
fread(&rootnum, sizeof(unsigned), 1, this->treefp);
fread(&cur_block_num, sizeof(unsigned), 1, this->treefp);
fseek(this->treefp, BLOCK_SIZE, SEEK_SET);
bp = this->freelist;
j = cur_block_num / 8;
for (i = 0; i < j; ++i)
{
c = fgetc(treefp);
for (k = 0; k < 8; ++k)
{
if ((c & (1 << k)) == 0)
{
bp->next = new BlockInfo(i * 8 + 7 - k + 1, NULL);
bp = bp->next;
}
}
}
fseek(treefp, Address(rootnum), SEEK_SET);
//treefp is now ahead of root-block
}
this->minheap = new IVHeap(this->heap_size);
this->value_list = _vlist;
}
bool
IVStorage::preRead(IVNode*& _root, IVNode*& _leaves_head, IVNode*& _leaves_tail) //pre-read and build whole tree
{ //set root(in memory) and leaves_head
//TODO: false when exceed memory
_leaves_tail = _leaves_head = _root = NULL;
if (ftell(this->treefp) == 0) //root is null
{
return true;
}
unsigned next, store, j, pos = 0;
unsigned h = *this->treeheight;
IVNode* p;
//read root node
this->createNode(p);
_root = p;
fread(&next, sizeof(unsigned), 1, treefp);
//use stack to achieve
long address[h]; //current address
unsigned used[h]; //used child num
unsigned total[h]; //total child num
unsigned block[h]; //next block num
IVNode* nodes[h];
address[pos] = ftell(treefp);
used[pos] = 0;
total[pos] = p->getNum() + 1;
block[pos] = next;
nodes[pos] = p;
pos++;
IVNode* prev = NULL;
while (pos > 0)
{
j = pos - 1;
if (nodes[j]->isLeaf() || used[j] == total[j]) //LeafNode or ready IntlNode
{
if (nodes[j]->isLeaf())
{
if (prev != NULL)
{
prev->setNext(nodes[j]);
nodes[j]->setPrev(prev);
}
prev = nodes[j];
}
pos--;
continue;
}
fseek(this->treefp, address[j], SEEK_SET);
fread(&store, sizeof(unsigned), 1, treefp);
this->ReadAlign(block + j);
address[j] = ftell(treefp);
fseek(treefp, Address(store), SEEK_SET);
this->createNode(p);
nodes[j]->setChild(p, used[j]);
used[j]++;
fread(&next, sizeof(unsigned), 1, treefp);
address[pos] = ftell(treefp);
used[pos] = 0;
total[pos] = p->getNum() + 1;
block[pos] = next;
nodes[pos] = p;
pos++;
}
//set leaves and read root, which is always keeped in-mem
p = _root;
while (!p->isLeaf())
{
p = p->getChild(0);
}
_leaves_head = p;
p = _root;
while (!p->isLeaf())
{
p = p->getChild(p->getNum());
}
_leaves_tail = p;
long long memory = 0;
this->readNode(_root, &memory);
this->request(memory);
return true;
}
long //8-byte in 64-bit machine
IVStorage::Address(unsigned _blocknum) const //BETTER: inline function
{
if (_blocknum == 0)
return 0;
else if (_blocknum > cur_block_num)
{
//print(string("error in Address: Invalid blocknum ") + Util::int2string(_blocknum));
return -1; //address should be non-negative
}
//NOTICE: here should explictly use long
return (long)(this->SuperNum + _blocknum - 1) * (long)BLOCK_SIZE;
}
unsigned
IVStorage::Blocknum(long address) const
{
return (address / BLOCK_SIZE) + 1 - this->SuperNum;
}
unsigned
IVStorage::AllocBlock()
{
BlockInfo* p = this->freelist->next;
if (p == NULL)
{
for (unsigned i = 0; i < SET_BLOCK_INC; ++i)
{
cur_block_num++; //BETTER: check if > MAX_BLOCK_NUM
this->FreeBlock(cur_block_num);
}
p = this->freelist->next;
}
unsigned t = p->num;
this->freelist->next = p->next;
delete p;
return t;
}
void
IVStorage::FreeBlock(unsigned _blocknum)
{ //QUERY: head-sub and tail-add will be better?
BlockInfo* bp = new BlockInfo(_blocknum, this->freelist->next);
this->freelist->next = bp;
}
//NOTICE: all reads are aligned to 4 bytes(including a string)
//a string may acrossseveral blocks
void
IVStorage::ReadAlign(unsigned* _next)
{
if (ftell(treefp) % BLOCK_SIZE == 0)
{
fseek(treefp, Address(*_next), SEEK_SET);
fread(_next, sizeof(unsigned), 1, treefp);
}
}
void
IVStorage::WriteAlign(unsigned* _curnum, bool& _SpecialBlock)
{
if (ftell(treefp) % BLOCK_SIZE == 0)
{
unsigned blocknum = this->AllocBlock();
fseek(treefp, Address(*_curnum), SEEK_SET);
if (_SpecialBlock)
{
fseek(treefp, 4, SEEK_CUR);
_SpecialBlock = false;
}
fwrite(&blocknum, sizeof(unsigned), 1, treefp);
fseek(treefp, Address(blocknum) + 4, SEEK_SET);
*_curnum = blocknum;
}
}
bool
IVStorage::readNode(IVNode* _np, long long* _request)
{
if (_np == NULL || _np->inMem())
return false; //can't read or needn't
fseek(treefp, Address(_np->getStore()), SEEK_SET);
bool flag = _np->isLeaf();
unsigned next;
unsigned i, num = _np->getNum();
Bstr bstr;
fseek(treefp, 4, SEEK_CUR);
fread(&next, sizeof(unsigned), 1, treefp);
//read data, use readBstr...
//fread(treefp, "%u", &num);
//_np->setNum(num);
if (flag)
*_request += IVNode::LEAF_SIZE;
else
*_request += IVNode::INTL_SIZE;
_np->Normal();
if (!flag)
fseek(treefp, 4 * (num + 1), SEEK_CUR);
//to read all keys
//int tmp = -1;
unsigned tmp = INVALID;
for (i = 0; i < num; ++i)
{
fread(&tmp, sizeof(int), 1, treefp);
this->ReadAlign(&next);
_np->setKey(tmp, i);
}
if (flag)
{
//to read all values
for (i = 0; i < num; ++i)
{
this->readBstr(&bstr, &next);
//if not long list value
if(bstr.getStr() != NULL)
{
*_request += bstr.getLen();
}
_np->setValue(&bstr, i);
}
}
//_np->setFlag((_np->getFlag() & ~Node::NF_IV & ~Node::NF_ID) | Node::NF_IM);
//_np->delVirtual();
_np->delDirty();
//_np->setMem();
this->updateHeap(_np, _np->getRank(), false);
bstr.clear();
return true;
}
bool
IVStorage::createNode(IVNode*& _np) //cretae virtual nodes, not in-mem
{
/*
if(ftell(this->treefp)== 0) //null root
{
_np = NULL;
return false;
}
*/
unsigned t; //QUERY: maybe next-flag... will be better-storage?
bool flag = false; //IntlNode
fread(&t, sizeof(unsigned), 1, treefp);
if ((t & IVNode::NF_IL) > 0) //WARN: according to setting
flag = true; //LeafNode
if (flag)
{
//this->request(sizeof(LeafNode));
_np = new IVLeafNode(true);
}
else
{
//this->request(sizeof(IntlNode));
_np = new IVIntlNode(true);
}
//fseek(treefp, -4, SEEK_CUR);
//_np->setFlag(_np->getFlag() | (t & Node::NF_RK));
//_np->setRank(t);
_np->setFlag(t);
_np->delDirty();
_np->delMem();
_np->setStore(Blocknum(ftell(treefp) - 4));
return true;
}
//BETTER: Does SpecialBlock really needed? why can't we place next before flag??
//
//NOTICE: root num begins from 1, if root num is 0, then it is invalid, i.e. the tree is NULL
//(and ftell(root address) will be 0 either)
bool
IVStorage::writeNode(IVNode* _np)
{
if (_np == NULL || !_np->inMem() || (_np->getRank() > 0 && !_np->isDirty()))
return false; //not need to write back
unsigned num = _np->getNum(), i;
bool flag = _np->isLeaf(), SpecialBlock = true;
/*
if(!flag)
{
for(i = 0; i <= num; ++i)
if(_np->getChild(i)->isDirty())
return false; //NOTICE: all childs must be clean!
}
*/
//to release original blocks
unsigned store = _np->getStore(), next;
//if first store is 0, meaning a new node
fseek(this->treefp, Address(store) + 4, SEEK_SET);
fread(&next, sizeof(unsigned), 1, treefp);
while (store != 0)
{
this->FreeBlock(store);
store = next;
fseek(treefp, Address(store), SEEK_SET);
fread(&next, sizeof(unsigned), 1, treefp);
}
if (num == 0)
return true; //node is empty!
unsigned t;
//write Node information
unsigned blocknum = this->AllocBlock();
_np->setStore(blocknum);
long address = this->Address(blocknum);
fseek(this->treefp, address, SEEK_SET);
t = _np->getFlag();
fwrite(&t, sizeof(unsigned), 1, treefp); //DEBUG
fseek(treefp, 4, SEEK_CUR);
if (!flag)
{
for (i = 0; i <= num; ++i)
{
t = _np->getChild(i)->getStore();
fwrite(&t, sizeof(unsigned), 1, treefp); //DEBUG
this->WriteAlign(&blocknum, SpecialBlock);
}
}
//int tmp = 0;
unsigned tmp = INVALID;
//to write all keys
for (i = 0; i < num; ++i)
{
tmp = _np->getKey(i);
fwrite(&tmp, sizeof(int), 1, treefp);
this->WriteAlign(&blocknum, SpecialBlock);
}
if (flag)
{
//to write all values
for (i = 0; i < num; ++i)
{
this->writeBstr(_np->getValue(i), &blocknum, SpecialBlock);
}
}
fseek(treefp, Address(blocknum), SEEK_SET);
if (SpecialBlock)
fseek(treefp, 4, SEEK_CUR);
t = 0;
fwrite(&t, sizeof(unsigned), 1, treefp); //the end-block
//_np->setFlag(_np->getFlag() & ~Node::NF_ID);
//NOTICE:we may store the dirty bit into the tree file, but that is ok
//Each time we read the tree file to construct a node, we always set the drity bit to 0
_np->delDirty();
return true;
}
bool
IVStorage::readBstr(Bstr* _bp, unsigned* _next)
{
//long address;
unsigned len, i, j;
fread(&len, sizeof(unsigned), 1, this->treefp);
this->ReadAlign(_next);
//NOTICE: if this is a long list as value
if(len == 0)
{
unsigned addr = 0;
fread(&addr, sizeof(unsigned), 1, this->treefp);
#ifdef DEBUG_VLIST
cout<<"read a vlist in IVStorage - addr: "<<addr<<endl;
#endif
_bp->setLen(addr);
_bp->setStr(NULL);
this->ReadAlign(_next);
return true;
}
//this->request(len);
//NOTICE: we use new for all, consistent with Bstr and KVstore
//char* s = (char*)malloc(len);
char* s = new char[len];
_bp->setLen(len);
for (i = 0; i + 4 < len; i += 4)
{
fread(s + i, sizeof(char), 4, treefp);
this->ReadAlign(_next);
}
while (i < len)
{
fread(s + i, sizeof(char), 1, treefp); //BETTER
i++;
}
j = len % 4;
if (j > 0)
j = 4 - j;
fseek(treefp, j, SEEK_CUR);
this->ReadAlign(_next);
_bp->setStr(s);
return true;
}
bool
IVStorage::writeBstr(const Bstr* _bp, unsigned* _curnum, bool& _SpecialBlock)
{
unsigned i, j, len = _bp->getLen();
//NOTICE: to write long list value
if(_bp->getStr() == NULL)
{
unsigned flag = 0;
fwrite(&flag, sizeof(unsigned), 1, treefp);
this->WriteAlign(_curnum, _SpecialBlock);
//then this is the real block num
fwrite(&len, sizeof(unsigned), 1, treefp);
#ifdef DEBUG_VLIST
cout<<"to write a vlist in IVStorage::writeBstr() - blocknum: "<<len<<endl;
#endif
this->WriteAlign(_curnum, _SpecialBlock);
return true;
}
fwrite(&len, sizeof(unsigned), 1, treefp);
this->WriteAlign(_curnum, _SpecialBlock);
char* s = _bp->getStr();
for (i = 0; i + 4 < len; i += 4)
{
fwrite(s + i, sizeof(char), 4, treefp);
this->WriteAlign(_curnum, _SpecialBlock);
}
while (i < len)
{
fwrite(s + i, sizeof(char), 1, treefp);
i++;
}
j = len % 4;
if (j > 0)
j = 4 - j;
fseek(treefp, j, SEEK_CUR);
this->WriteAlign(_curnum, _SpecialBlock);
return true;
}
bool
IVStorage::writeTree(IVNode* _root) //write the whole tree back and close treefp
{
fseek(this->treefp, 0, SEEK_SET);
fwrite(this->treeheight, sizeof(unsigned), 1, treefp);
//delete all nonsense-node in heap, otherwise will waste storage permanently
IVNode* p;
while (1)
{ //all non-sense nodes will be in-head-area, due to minimal rank
p = minheap->getTop();
if (p == NULL) //heap is empty, only when root==NULL
break;
if (p->getRank() == 0) //indicate non-sense node
{
this->minheap->remove();
this->writeNode(p);
delete p;
}
else
break;
}
unsigned i, j, t;
//QUERY: another way to write all nodes back is to print out all nodes in heap
//but this method will cause no node in heap any more, while operations may be
//afetr tree-saving. Which method is better?
//write nodes recursively using stack, including root-num
if (_root != NULL)
{
IVNode* p = _root;
unsigned h = *this->treeheight, pos = 0;
IVNode* ns[h];
int ni[h];
ns[pos] = p;
ni[pos] = p->getNum();
pos++;
while (pos > 0)
{
j = pos - 1;
p = ns[j];
if (p->isLeaf() || ni[j] < 0) //leaf or all childs are ready
{
this->writeNode(p);
pos--;
continue;
}
ns[pos] = p->getChild(ni[j]);
ni[pos] = ns[pos]->getNum();
pos++;
ni[j]--;
}
t = _root->getStore();
}
else
t = 0;
fseek(this->treefp, 4, SEEK_SET);
fwrite(&t, sizeof(unsigned), 1, treefp); //write the root num
fwrite(&cur_block_num, sizeof(unsigned), 1, treefp);//write current blocks num
fseek(treefp, BLOCK_SIZE, SEEK_SET);
j = cur_block_num / 8; //(SuperNum-1)*BLOCK_SIZE;
//reset to 1 first
for (i = 0; i < j; ++i)
{
fputc(0xff, treefp);
}
char c;
BlockInfo* bp = this->freelist->next;
while (bp != NULL)
{
//if not-use then set 0, aligned to byte!
#ifdef DEBUG_KVSTORE
if (bp->num > cur_block_num)
{
printf("blocks num exceed, cur_block_num: %u\n", cur_block_num);
exit(1);
}
#endif
j = bp->num - 1;
i = j / 8;
j = 7 - j % 8;
fseek(treefp, BLOCK_SIZE + i, SEEK_SET);
c = fgetc(treefp);
fseek(treefp, -1, SEEK_CUR);
fputc(c & ~(1 << j), treefp);
bp = bp->next;
}
fflush(this->treefp);
//fclose(this->treefp);
return true;
}
void
IVStorage::updateHeap(IVNode* _np, unsigned _rank, bool _inheap) const
{
if (_inheap) //already in heap, to modify
{
unsigned t = _np->getRank();
_np->setRank(_rank);
if (t < _rank)
this->minheap->modify(_np, false);
else if (t > _rank)
this->minheap->modify(_np, true);
else;
}
else //not in heap, to add
{
_np->setRank(_rank);
this->minheap->insert(_np);
}
}
bool
IVStorage::request(long long _needmem) //aligned to byte
{ //NOTICE: <0 means release
//cout<<"freemem: "<<this->freemem<<" needmem: "<<_needmem<<endl;
if (_needmem > 0 && this->freemem < (unsigned long long)_needmem)
if (!this->handler(_needmem - freemem)) //disaster in buffer memory
{
print(string("error in request: out of buffer-mem, now to exit"));
//exit(1);
return false;
}
this->freemem -= _needmem;
return true;
}
bool
IVStorage::handler(unsigned long long _needmem) //>0
{
//cout<<"swap happen"<<endl;
IVNode* p;
unsigned long long size;
//if(_needmem < SET_BUFFER_SIZE) //to recover to SET_BUFFER_SIZE buffer
// _needmem = SET_BUFFER_SIZE;
//cout<<"IVStorage::handler() - now to loop to release nodes"<<endl;
while (1)
{
p = this->minheap->getTop();
//cout<<"get heap top"<<endl;
if (p == NULL)
{
cout << "the heap top is null" << endl;
return false; //can't satisfy or can't recover to SET_BUFFER_SIZE
}
this->minheap->remove();
//cout<<"node removed in heap"<<endl;
size = p->getSize();
this->freemem += size;
this->writeNode(p);
//cout<<"node write back"<<endl;
if (p->getNum() > 0)
p->Virtual();
else
delete p; //non-sense node
//cout<<"node memory released"<<endl;
if (_needmem > size)
{
//cout<<"reduce the request"<<endl;
_needmem -= size;
}
else
{
//cout<<"ok to break"<<endl;
break;
}
}
//cout<<"IVStorage::handler() -- finished"<<endl;
return true;
}
IVStorage::~IVStorage()
{
//release heap and freelist...
#ifdef DEBUG_KVSTORE
printf("now to release the kvstore!\n");
#endif
BlockInfo* bp = this->freelist;
BlockInfo* next;
while (bp != NULL)
{
next = bp->next;
delete bp;
bp = next;
}
#ifdef DEBUG_KVSTORE
printf("already empty the freelist!\n");
#endif
delete this->minheap;
this->minheap = NULL;
#ifdef DEBUG_KVSTORE
printf("already empty the buffer heap!\n");
#endif
fclose(this->treefp);
//#ifdef DEBUG_KVSTORE
//NOTICE:there is more than one tree
//fclose(Util::debug_kvstore); //NULL is ok!
//Util::debug_kvstore = NULL;
//#endif
}
void
IVStorage::print(string s)
{
#ifdef DEBUG_KVSTORE
fputs(Util::showtime().c_str(), Util::debug_kvstore);
fputs("Class IVStorage\n", Util::debug_kvstore);
fputs("Message: ", Util::debug_kvstore);
fputs(s.c_str(), Util::debug_kvstore);
fputs("\n", Util::debug_kvstore);
#endif
}

84
KVstore/IVTree/storage/IVStorage.h Normal file
View File

@ -0,0 +1,84 @@
/*=============================================================================
# Filename: IVStorage.h
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:43
# Description: swap between memory and disk, achieving system-like method
=============================================================================*/
#ifndef _KVSTORE_IVTREE_STORAGE_IVSTORAGE_H
#define _KVSTORE_IVTREE_STORAGE_IVSTORAGE_H
#include "../../../Util/VList.h"
#include "../node/IVIntlNode.h"
#include "../node/IVLeafNode.h"
#include "../heap/IVHeap.h"
//It controls read, write, swap
class IVStorage
{
public:
static const unsigned BLOCK_SIZE = Util::STORAGE_BLOCK_SIZE; //fixed size of disk-block
//there are 18 B+Tree indexes and one vstree index, so set 3G buffer size
//static const unsigned long long MAX_BUFFER_SIZE = Util::MAX_BUFFER_SIZE; //max buffer size
//static const unsigned SET_BUFFER_SIZE = 1 << 30; //set buffer size
//static const unsigned HEAP_SIZE = MAX_BUFFER_SIZE / IVNode::INTL_SIZE;
//DEBUG: maybe need to set larger, now the file size is 64G at most
static const unsigned MAX_BLOCK_NUM = 1 << 24; //max block-num
//below two constants: must can be exactly divided by 8
static const unsigned SET_BLOCK_NUM = 1 << 8; //initial blocks num
static const unsigned SET_BLOCK_INC = SET_BLOCK_NUM; //base of blocks-num inc
static const unsigned SuperNum = MAX_BLOCK_NUM / (8 * BLOCK_SIZE) + 1;
//static const unsigned TRANSFER_CAPACITY = BLOCK_SIZE;
//enum ReadType { OVER = 0, EXPAND, NORMAL };
private:
unsigned long long max_buffer_size;
unsigned heap_size;
unsigned cur_block_num;
std::string filepath;
unsigned* treeheight;
BlockInfo* freelist;
FILE* treefp; //file: tree nodes
IVHeap* minheap; //heap of Nodes's pointer, sorted in NF_RK
//very long value list are stored in a separate file(with large block)
//
//NOTICE: according to the summary result, 90% value lists are just below 100 bytes
//<10%: 5000000~100M bytes
VList* value_list;
//NOTICE: freemem's type is long long here, due to large memory in server.
//However, needmem in handler() and request() is ok to be int/unsigned.
//Because the bstr' size is controlled, so is the node.
unsigned long long freemem; //free memory to use, non-negative
//unsigned long long time; //QUERY(achieving an old-swap startegy?)
long Address(unsigned _blocknum) const;
unsigned Blocknum(long address) const;
unsigned AllocBlock();
void FreeBlock(unsigned _blocknum);
void ReadAlign(unsigned* _next);
void WriteAlign(unsigned* _next, bool& _SpecialBlock);
public:
IVStorage();
IVStorage(std::string& _filepath, std::string& _mode, unsigned* _height, unsigned long long _buffer_size, VList* _vlist); //create a fixed-size file or open an existence
bool preRead(IVNode*& _root, IVNode*& _leaves_head, IVNode*& _leaves_tail); //read and build all nodes, only root in memory
bool readNode(IVNode* _np, long long* _request); //read, if virtual
bool createNode(IVNode*& _np); //use fp to create a new node
//NOTICE(if children and child not exist, build children's Nodes)
bool writeNode(IVNode* _np);
bool readBstr(Bstr* _bp, unsigned* _next);
bool writeBstr(const Bstr* _bp, unsigned* _curnum, bool& _SpecialBlock);
bool writeTree(IVNode* _np);
void updateHeap(IVNode* _np, unsigned _rank, bool _inheap) const;
bool request(long long _needmem); //deal with memory request
bool handler(unsigned long long _needmem); //swap some nodes out
//bool update(); //update InMem Node's rank, with clock
~IVStorage();
void print(std::string s); //DEBUG
};
#endif

4492
KVstore/KVstore.cpp
View File

File diff suppressed because it is too large Load Diff

456
KVstore/KVstore.h
View File

@ -1,207 +1,249 @@
/*=============================================================================
# Filename: KVstore.h
# Author: Bookug Lobert
# Mail: 1181955272@qq.com
# Last Modified: 2015-10-23 14:23
# Description: Modified by Wang Libo
=============================================================================*/
#ifndef _KVSTORE_KVSTORE_H
#define _KVSTORE_KVSTORE_H
#include "../Util/Util.h"
#include "Tree.h"
class KVstore
{
public:
static const int READ_WRITE_MODE = 1; //Open a B tree, which must exist
static const int CREATE_MODE = 2; //Build a new B tree and delete existing ones (if any)
KVstore(std::string _store_path = ".");
~KVstore();
void flush();
void release();
void open();
//===============================================================================
//including IN-neighbor & OUT-neighbor
int getEntityDegree(int _entity_id) const;
int getEntityInDegree(int _entity_id) const;
int getEntityOutDegree(int _entity_id) const;
int getLiteralDegree(int _literal_id) const;
int getPredicateDegree(int _predicate_id) const;
int getSubjectPredicateDegree(int _subid, int _preid) const;
int getObjectPredicateDegree(int _objid, int _preid) const;
//===============================================================================
//Before calling these functions, we are sure that the triples doesn't exist.
bool updateTupleslist_insert(int _sub_id, int _pre_id, int _obj_id);
bool updateTupleslist_remove(int _sub_id, int _pre_id, int _obj_id);
bool updateInsert_s2values(int _sub_id, int _pre_id, int _obj_id);
bool updateRemove_s2values(int _sub_id, int _pre_id, int _obj_id);
bool updateInsert_s2values(int _subid, const std::vector<int>& _pidoidlist);
bool updateRemove_s2values(int _subid, const std::vector<int>& _pidoidlist);
bool updateInsert_o2values(int _sub_id, int _pre_id, int _obj_id);
bool updateRemove_o2values(int _sub_id, int _pre_id, int _obj_id);
bool updateInsert_o2values(int _objid, const std::vector<int>& _pidsidlist);
bool updateRemove_o2values(int _objid, const std::vector<int>& _pidsidlist);
bool updateInsert_p2values(int _sub_id, int _pre_id, int _obj_id);
bool updateRemove_p2values(int _sub_id, int _pre_id, int _obj_id);
bool updateInsert_p2values(int _preid, const std::vector<int>& _sidoidlist);
bool updateRemove_p2values(int _preid, const std::vector<int>& _sidoidlist);
//===============================================================================
//for entity2id
bool open_entity2id(int _mode);
bool close_entity2id();
bool subIDByEntity(std::string _entity);
int getIDByEntity(std::string _entity) const;
bool setIDByEntity(std::string _entity, int _id);
//for id2entity
bool open_id2entity(int _mode);
bool close_id2entity();
bool subEntityByID(int _id);
std::string getEntityByID(int _id) const;
bool setEntityByID(int _id, std::string _entity);
//for predicate2id
bool open_predicate2id(int _mode);
bool close_predicate2id();
bool subIDByPredicate(std::string _predicate);
int getIDByPredicate(std::string _predicate) const;
bool setIDByPredicate(std::string _predicate, int _id);
//for id2predicate
bool open_id2predicate(int _mode);
bool close_id2predicate();
bool subPredicateByID(int _id);
std::string getPredicateByID(int _id) const;
bool setPredicateByID(int _id, std::string _predicate);
//for literal2id
bool open_literal2id(int _mode);
bool close_literal2id();
bool subIDByLiteral(std::string _literal);
int getIDByLiteral(std::string _literal) const;
bool setIDByLiteral(std::string _literal, int _id);
//for id2literal
bool open_id2literal(int _mode);
bool close_id2literal();
bool subLiteralByID(int _id);
std::string getLiteralByID(int _id) const;
bool setLiteralByID(int _id, std::string _literal);
//===============================================================================
//for subID2values
bool open_subID2values(int _mode);
bool close_subID2values();
bool build_subID2values(int** _p_id_tuples, int _triples_num);
bool getpreIDlistBysubID(int _subid, int*& _preidlist, int& _list_len, bool _no_duplicate = false) const;
bool getobjIDlistBysubID(int _subid, int*& _objidlist, int& _list_len, bool _no_duplicate = false) const;
bool getobjIDlistBysubIDpreID(int _subid, int _preid, int*& _objidlist, int& _list_len, bool _no_duplicate = false) const;
bool getpreIDobjIDlistBysubID(int _subid, int*& _preid_objidlist, int& _list_len, bool _no_duplicate = false) const;
//for objID2values
bool open_objID2values(int _mode);
bool close_objID2values();
bool build_objID2values(int** _p_id_tuples, int _triples_num);
bool getpreIDlistByobjID(int _objid, int*& _preidlist, int& _list_len, bool _no_duplicate = false) const;
bool getsubIDlistByobjID(int _objid, int*& _subidlist, int& _list_len, bool _no_duplicate = false) const;
bool getsubIDlistByobjIDpreID(int _objid, int _preid, int*& _subidlist, int& _list_len, bool _no_duplicate = false) const;
bool getpreIDsubIDlistByobjID(int _objid, int*& _preid_subidlist, int& _list_len, bool _no_duplicate = false) const;
//for preID2values
bool open_preID2values(int _mode);
bool close_preID2values();
bool build_preID2values(int** _p_id_tuples, int _triples_num);
bool getsubIDlistBypreID(int _preid, int*& _subidlist, int& _list_len, bool _no_duplicate = false) const;
bool getobjIDlistBypreID(int _preid, int*& _objidlist, int& _list_len, bool _no_duplicate = false) const;
bool getsubIDobjIDlistBypreID(int _preid, int*& _subid_objidlist, int& _list_len, bool _no_duplicate = false) const;
//for so2p
bool getpreIDlistBysubIDobjID(int _subID, int _objID, int*& _preidlist, int& _list_len, bool _no_duplicate = false) const;
private:
std::string store_path;
SITree* entity2id;
ISTree* id2entity;
static std::string s_entity2id;
static std::string s_id2entity;
static unsigned short buffer_entity2id_build;
static unsigned short buffer_id2entity_build;
static unsigned short buffer_entity2id_query;
static unsigned short buffer_id2entity_query;
SITree* predicate2id;
ISTree* id2predicate;
static std::string s_predicate2id;
static std::string s_id2predicate;
static unsigned short buffer_predicate2id_build;
static unsigned short buffer_id2predicate_build;
static unsigned short buffer_predicate2id_query;
static unsigned short buffer_id2predicate_query;
SITree* literal2id;
ISTree* id2literal;
static std::string s_literal2id;
static std::string s_id2literal;
static unsigned short buffer_literal2id_build;
static unsigned short buffer_id2literal_build;
static unsigned short buffer_literal2id_query;
static unsigned short buffer_id2literal_query;
ISTree* subID2values;
ISTree* objID2values;
ISTree* preID2values;
static std::string s_sID2values;
static std::string s_oID2values;
static std::string s_pID2values;
static unsigned short buffer_sID2values_build;
static unsigned short buffer_oID2values_build;
static unsigned short buffer_pID2values_build;
static unsigned short buffer_sID2values_query;
static unsigned short buffer_oID2values_query;
static unsigned short buffer_pID2values_query;
//===============================================================================
bool open(SITree* & _p_btree, std::string _tree_name, int _mode, unsigned long long _buffer_size);
bool open(ISTree* & _p_btree, std::string _tree_name, int _mode, unsigned long long _buffer_size);
void flush(SITree* _p_btree);
void flush(ISTree* _p_btree);
bool addValueByKey(SITree* _p_btree, const char* _key, int _klen, int _val);
bool addValueByKey(ISTree* _p_btree, int _key, const char* _val, int _vlen);
bool setValueByKey(SITree* _p_btree, const char* _key, int _klen, int _val);
bool setValueByKey(ISTree* _p_btree, int _key, const char* _val, int _vlen);
bool getValueByKey(SITree* _p_btree, const char* _key, int _klen, int* _val) const;
bool getValueByKey(ISTree* _p_btree, int _key, char*& _val, int& _vlen) const;
int getIDByStr(SITree* _p_btree, const char* _key, int _klen) const;
bool removeKey(SITree* _p_btree, const char* _key, int _klen);
bool removeKey(ISTree* _p_btree, int _key);
static std::vector<int> intersect(const int* _list1, const int* _list2, int _len1, int _len2);
static int binarySearch(int key, const int* _list, int _list_len, int step = 1);
static bool isEntity(int id);
};
#endif //_KVSTORE_KVSTORE_H
/*=============================================================================
# Filename: KVstore.h
# Author: Bookug Lobert
# Mail: 1181955272@qq.com
# Last Modified: 2015-10-23 14:23
# Description: Modified by Wang Libo
=============================================================================*/
#ifndef _KVSTORE_KVSTORE_H
#define _KVSTORE_KVSTORE_H
#include "../Util/Util.h"
#include "../Util/VList.h"
#include "Tree.h"
//TODO: is it needed to keep a length in Bstr?? especially for IVTree?
//add a length: sizeof bstr from 8 to 16(4 -> 8 for alignment)
//add a \0 in tail: only add 1 char
//QUERY: but to count the length each time maybe very costly?
//No, because triple num is stored in char* now!!!! we do not need to save it again
//
//QUERY: but to implement vlist, we need a unsigned flag
//What is more, we need to store the string in disk, how can we store it if without the length?
//unsigned type stored as chars, maybe will have '\0'
//In memory, we do not know when the oidlist ends if without the original length (butthe triple num will answer this!)
//
//TODO: entity_border in s2values list is not needed!!! not waste memory here
//STRUCT:
//1. s2xx
//Triple Num Pre Num Entity Num p1 offset1 p2 offset2 ... pn offsetn (olist-p1) (olist-p2) ... (olist-pn)
//2. o2xx
//Triple Num Pre Num p1 offset1 p2 offset2 ... pn offsetn (slist-p1) (slist-p2) ... (slist-pn)
//3. p2xx
//Triple Num (sid list) (oid list) (not sorted, matched with sid one by one)
class KVstore
{
public:
static const int READ_WRITE_MODE = 1; //Open a B tree, which must exist
static const int CREATE_MODE = 2; //Build a new B tree and delete existing ones (if any)
KVstore(std::string _store_path = ".");
~KVstore();
void flush();
void release();
void open();
std::string getStringByID(TYPE_ENTITY_LITERAL_ID _id);
TYPE_ENTITY_LITERAL_ID getIDByString(std::string _str);
// functions to load Vlist's cache
void AddIntoSubCache(TYPE_ENTITY_LITERAL_ID _entity_id);
void AddIntoObjCache(TYPE_ENTITY_LITERAL_ID _entity_literal_id);
void AddIntoPreCache(TYPE_PREDICATE_ID _pre_id);
unsigned getPreListSize(TYPE_PREDICATE_ID _pre_id);
unsigned getSubListSize(TYPE_ENTITY_LITERAL_ID _sub_id);
unsigned getObjListSize(TYPE_ENTITY_LITERAL_ID _obj_id);
//===============================================================================
//including IN-neighbor & OUT-neighbor
unsigned getEntityDegree(TYPE_ENTITY_LITERAL_ID _entity_id) const;
unsigned getEntityInDegree(TYPE_ENTITY_LITERAL_ID _entity_id) const;
unsigned getEntityOutDegree(TYPE_ENTITY_LITERAL_ID _entity_id) const;
unsigned getLiteralDegree(TYPE_ENTITY_LITERAL_ID _literal_id) const;
unsigned getPredicateDegree(TYPE_PREDICATE_ID _predicate_id) const;
unsigned getSubjectPredicateDegree(TYPE_ENTITY_LITERAL_ID _subid, TYPE_PREDICATE_ID _preid) const;
unsigned getObjectPredicateDegree(TYPE_ENTITY_LITERAL_ID _objid, TYPE_PREDICATE_ID _preid) const;
//===============================================================================
//Before calling these functions, we are sure that the triples doesn't exist.
bool updateTupleslist_insert(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
bool updateTupleslist_remove(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
bool updateInsert_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
bool updateRemove_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
bool updateInsert_s2values(TYPE_ENTITY_LITERAL_ID _subid, const std::vector<unsigned>& _pidoidlist);
bool updateRemove_s2values(TYPE_ENTITY_LITERAL_ID _subid, const std::vector<unsigned>& _pidoidlist);
bool updateInsert_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
bool updateRemove_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
bool updateInsert_o2values(TYPE_ENTITY_LITERAL_ID _objid, const std::vector<unsigned>& _pidsidlist);
bool updateRemove_o2values(TYPE_ENTITY_LITERAL_ID _objid, const std::vector<unsigned>& _pidsidlist);
bool updateInsert_p2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
bool updateRemove_p2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id);
bool updateInsert_p2values(TYPE_PREDICATE_ID _preid, const std::vector<unsigned>& _sidoidlist);
bool updateRemove_p2values(TYPE_PREDICATE_ID _preid, const std::vector<unsigned>& _sidoidlist);
//===============================================================================
//for entity2id
bool open_entity2id(int _mode);
bool close_entity2id();
bool subIDByEntity(std::string _entity);
TYPE_ENTITY_LITERAL_ID getIDByEntity(std::string _entity) const;
bool setIDByEntity(std::string _entity, TYPE_ENTITY_LITERAL_ID _id);
//for id2entity
bool open_id2entity(int _mode);
bool close_id2entity();
bool subEntityByID(TYPE_ENTITY_LITERAL_ID _id);
std::string getEntityByID(TYPE_ENTITY_LITERAL_ID _id) const;
bool setEntityByID(TYPE_ENTITY_LITERAL_ID _id, std::string _entity);
//for predicate2id
bool open_predicate2id(int _mode);
bool close_predicate2id();
bool subIDByPredicate(std::string _predicate);
TYPE_PREDICATE_ID getIDByPredicate(std::string _predicate) const;
bool setIDByPredicate(std::string _predicate, TYPE_PREDICATE_ID _id);
//for id2predicate
bool open_id2predicate(int _mode);
bool close_id2predicate();
bool subPredicateByID(TYPE_PREDICATE_ID _id);
std::string getPredicateByID(TYPE_PREDICATE_ID _id) const;
bool setPredicateByID(TYPE_PREDICATE_ID _id, std::string _predicate);
//for literal2id
bool open_literal2id(int _mode);
bool close_literal2id();
bool subIDByLiteral(std::string _literal);
TYPE_ENTITY_LITERAL_ID getIDByLiteral(std::string _literal) const;
bool setIDByLiteral(std::string _literal, TYPE_ENTITY_LITERAL_ID _id);
//for id2literal
bool open_id2literal(int _mode);
bool close_id2literal();
bool subLiteralByID(TYPE_ENTITY_LITERAL_ID _id);
std::string getLiteralByID(TYPE_ENTITY_LITERAL_ID _id) const;
bool setLiteralByID(TYPE_ENTITY_LITERAL_ID _id, std::string _literal);
//===============================================================================
//for subID2values
bool open_subID2values(int _mode);
bool close_subID2values();
bool build_subID2values(ID_TUPLE* _p_id_tuples, TYPE_TRIPLE_NUM _triples_num);
bool getpreIDlistBysubID(TYPE_ENTITY_LITERAL_ID _subid, unsigned*& _preidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool getobjIDlistBysubID(TYPE_ENTITY_LITERAL_ID _subid, unsigned*& _objidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool getobjIDlistBysubIDpreID(TYPE_ENTITY_LITERAL_ID _subid, TYPE_PREDICATE_ID _preid, unsigned*& _objidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool getpreIDobjIDlistBysubID(TYPE_ENTITY_LITERAL_ID _subid, unsigned*& _preid_objidlist, unsigned& _list_len, bool _no_duplicate = false) const;
//for objID2values
bool open_objID2values(int _mode);
bool close_objID2values();
bool build_objID2values(ID_TUPLE* _p_id_tuples, TYPE_TRIPLE_NUM _triples_num);
bool getpreIDlistByobjID(TYPE_ENTITY_LITERAL_ID _objid, unsigned*& _preidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool getsubIDlistByobjID(TYPE_ENTITY_LITERAL_ID _objid, unsigned*& _subidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool getsubIDlistByobjIDpreID(TYPE_ENTITY_LITERAL_ID _objid, TYPE_PREDICATE_ID _preid, unsigned*& _subidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool getpreIDsubIDlistByobjID(TYPE_ENTITY_LITERAL_ID _objid, unsigned*& _preid_subidlist, unsigned& _list_len, bool _no_duplicate = false) const;
//for preID2values
bool open_preID2values(int _mode);
bool close_preID2values();
bool build_preID2values(ID_TUPLE* _p_id_tuples, TYPE_TRIPLE_NUM _triples_num);
bool getsubIDlistBypreID(TYPE_PREDICATE_ID _preid, unsigned*& _subidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool getobjIDlistBypreID(TYPE_PREDICATE_ID _preid, unsigned*& _objidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool getsubIDobjIDlistBypreID(TYPE_PREDICATE_ID _preid, unsigned*& _subid_objidlist, unsigned& _list_len, bool _no_duplicate = false) const;
//for so2p
bool getpreIDlistBysubIDobjID(TYPE_ENTITY_LITERAL_ID _subID, TYPE_ENTITY_LITERAL_ID _objID, unsigned*& _preidlist, unsigned& _list_len, bool _no_duplicate = false) const;
private:
std::string store_path;
SITree* entity2id;
ISTree* id2entity;
static std::string s_entity2id;
static std::string s_id2entity;
static unsigned short buffer_entity2id_build;
static unsigned short buffer_id2entity_build;
static unsigned short buffer_entity2id_query;
static unsigned short buffer_id2entity_query;
SITree* predicate2id;
ISTree* id2predicate;
static std::string s_predicate2id;
static std::string s_id2predicate;
static unsigned short buffer_predicate2id_build;
static unsigned short buffer_id2predicate_build;
static unsigned short buffer_predicate2id_query;
static unsigned short buffer_id2predicate_query;
SITree* literal2id;
ISTree* id2literal;
static std::string s_literal2id;
static std::string s_id2literal;
static unsigned short buffer_literal2id_build;
static unsigned short buffer_id2literal_build;
static unsigned short buffer_literal2id_query;
static unsigned short buffer_id2literal_query;
IVTree* subID2values;
IVTree* objID2values;
IVTree* preID2values;
static std::string s_sID2values;
static std::string s_oID2values;
static std::string s_pID2values;
static unsigned short buffer_sID2values_build;
static unsigned short buffer_oID2values_build;
static unsigned short buffer_pID2values_build;
static unsigned short buffer_sID2values_query;
static unsigned short buffer_oID2values_query;
static unsigned short buffer_pID2values_query;
//===============================================================================
bool open(SITree* & _p_btree, std::string _tree_name, int _mode, unsigned long long _buffer_size);
bool open(ISTree* & _p_btree, std::string _tree_name, int _mode, unsigned long long _buffer_size);
bool open(IVTree* & _p_btree, std::string _tree_name, int _mode, unsigned long long _buffer_size);
void flush(SITree* _p_btree);
void flush(ISTree* _p_btree);
void flush(IVTree* _p_btree);
bool addValueByKey(SITree* _p_btree, char* _key, unsigned _klen, unsigned _val);
bool addValueByKey(ISTree* _p_btree, unsigned _key, char* _val, unsigned _vlen);
bool addValueByKey(IVTree* _p_btree, unsigned _key, char* _val, unsigned _vlen);
bool setValueByKey(SITree* _p_btree, char* _key, unsigned _klen, unsigned _val);
bool setValueByKey(ISTree* _p_btree, unsigned _key, char* _val, unsigned _vlen);
bool setValueByKey(IVTree* _p_btree, unsigned _key, char* _val, unsigned _vlen);
bool getValueByKey(SITree* _p_btree, const char* _key, unsigned _klen, unsigned* _val) const;
bool getValueByKey(ISTree* _p_btree, unsigned _key, char*& _val, unsigned& _vlen) const;
bool getValueByKey(IVTree* _p_btree, unsigned _key, char*& _val, unsigned& _vlen) const;
TYPE_ENTITY_LITERAL_ID getIDByStr(SITree* _p_btree, const char* _key, unsigned _klen) const;
bool removeKey(SITree* _p_btree, const char* _key, unsigned _klen);
bool removeKey(ISTree* _p_btree, unsigned _key);
bool removeKey(IVTree* _p_btree, unsigned _key);
static std::vector<unsigned> intersect(const unsigned* _list1, const unsigned* _list2, unsigned _len1, unsigned _len2);
static unsigned binarySearch(unsigned key, const unsigned* _list, unsigned _list_len, int step = 1);
static bool isEntity(TYPE_ENTITY_LITERAL_ID id);
};
#endif //_KVSTORE_KVSTORE_H

183
KVstore/SITree/SITree.cpp
View File

@ -20,7 +20,7 @@ SITree::SITree()
TSM = NULL;
storepath = "";
filename = "";
transfer_size[0] = transfer_size[1] = transfer_size[2] = 0;
//transfer_size[0] = transfer_size[1] = transfer_size[2] = 0;
this->request = 0;
}
@ -36,10 +36,10 @@ SITree::SITree(string _storepath, string _filename, string _mode, unsigned long
this->TSM->preRead(this->root, this->leaves_head, this->leaves_tail);
else
this->root = NULL;
this->transfer[0].setStr((char*)malloc(Util::TRANSFER_SIZE));
this->transfer[1].setStr((char*)malloc(Util::TRANSFER_SIZE));
this->transfer[2].setStr((char*)malloc(Util::TRANSFER_SIZE));
this->transfer_size[0] = this->transfer_size[1] = this->transfer_size[2] = Util::TRANSFER_SIZE; //initialied to 1M
//this->transfer[0].setStr((char*)malloc(Util::TRANSFER_SIZE));
//this->transfer[1].setStr((char*)malloc(Util::TRANSFER_SIZE));
//this->transfer[2].setStr((char*)malloc(Util::TRANSFER_SIZE));
//this->transfer_size[0] = this->transfer_size[1] = this->transfer_size[2] = Util::TRANSFER_SIZE; //initialied to 1M
this->request = 0;
}
@ -49,30 +49,30 @@ SITree::getFilePath()
return storepath + "/" + filename;
}
void //WARN: not check _str and _len
SITree::CopyToTransfer(const char* _str, unsigned _len, unsigned _index)
{
if (_index > 2)
return;
/*
if(_str == NULL || _len == 0)
{
printf("error in CopyToTransfer: empty string\n");
return;
}
*/
//unsigned length = _bstr->getLen();
unsigned length = _len;
if (length + 1 > this->transfer_size[_index])
{
transfer[_index].release();
transfer[_index].setStr((char*)malloc(length + 1));
this->transfer_size[_index] = length + 1; //one more byte: convenient to add \0
}
memcpy(this->transfer[_index].getStr(), _str, length);
this->transfer[_index].getStr()[length] = '\0'; //set for string() in KVstore
this->transfer[_index].setLen(length);
}
//void //WARN: not check _str and _len
//SITree::CopyToTransfer(const char* _str, unsigned _len, unsigned _index)
//{
//if (_index > 2)
//return;
//[>
//if(_str == NULL || _len == 0)
//{
//printf("error in CopyToTransfer: empty string\n");
//return;
//}
//*/
////unsigned length = _bstr->getLen();
//unsigned length = _len;
//if (length + 1 > this->transfer_size[_index])
//{
//transfer[_index].release();
//transfer[_index].setStr((char*)malloc(length + 1));
//this->transfer_size[_index] = length + 1; //one more byte: convenient to add \0
//}
//memcpy(this->transfer[_index].getStr(), _str, length);
//this->transfer[_index].getStr()[length] = '\0'; //set for string() in KVstore
//this->transfer[_index].setLen(length);
//}
unsigned
SITree::getHeight() const
@ -102,40 +102,47 @@ SITree::prepare(SINode* _np)
}
bool
SITree::search(const char* _str, unsigned _len, int* _val)
SITree::search(const char* _str, unsigned _len, unsigned* _val)
{
if (_str == NULL || _len == 0)
{
printf("error in SITree-search: empty string\n");
*_val = -1;
//*_val = -1;
return false;
}
this->CopyToTransfer(_str, _len, 1);
//this->CopyToTransfer(_str, _len, 1);
request = 0;
Bstr bstr = this->transfer[1]; //not to modify its memory
//Bstr bstr = this->transfer[1]; //not to modify its memory
//Bstr bstr(_str, _len, true);
int store;
SINode* ret = this->find(&transfer[1], &store, false);
if (ret == NULL || store == -1 || bstr != *(ret->getKey(store))) //tree is empty or not found
SINode* ret = this->find(_str, _len, &store, false);
if (ret == NULL || store == -1) //tree is empty or not found
{
//bstr.clear();
return false;
}
const Bstr* tmp = ret->getKey(store);
if (Util::compare(_str, _len, tmp->getStr(), tmp->getLen()) != 0) //tree is empty or not found
{
bstr.clear();
return false;
}
*_val = ret->getValue(store);
this->TSM->request(request);
bstr.clear();
//bstr.clear();
return true;
}
bool
SITree::insert(const char* _str, unsigned _len, int _val)
SITree::insert(char* _str, unsigned _len, unsigned _val)
{
if (_str == NULL || _len == 0)
{
printf("error in SITree-insert: empty string\n");
return false;
}
this->CopyToTransfer(_str, _len, 1);
//this->CopyToTransfer(_str, _len, 1);
this->request = 0;
SINode* ret;
@ -170,8 +177,8 @@ SITree::insert(const char* _str, unsigned _len, int _val)
SINode* p = this->root;
SINode* q;
int i;
const Bstr* _key = &transfer[1];
Bstr bstr = *_key;
//const Bstr* _key = &transfer[1];
//Bstr bstr = *_key;
while (!p->isLeaf())
{
//j = p->getNum();
@ -179,7 +186,7 @@ SITree::insert(const char* _str, unsigned _len, int _val)
//if(bstr < *(p->getKey(i)))
//break;
//NOTICE: using binary search is better here
i = p->searchKey_less(bstr);
i = p->searchKey_less(_str, _len);
q = p->getChild(i);
this->prepare(q);
@ -196,7 +203,10 @@ SITree::insert(const char* _str, unsigned _len, int _val)
this->TSM->updateHeap(ret, ret->getRank(), false);
this->TSM->updateHeap(q, q->getRank(), true);
this->TSM->updateHeap(p, p->getRank(), true);
if (bstr < *(p->getKey(i)))
//if (bstr < *(p->getKey(i)))
const Bstr* tmp = p->getKey(i);
int cmp_res = Util::compare(_str, _len, tmp->getStr(), tmp->getLen());
if (cmp_res < 0)
p = q;
else
p = ret;
@ -212,63 +222,82 @@ SITree::insert(const char* _str, unsigned _len, int _val)
//for(i = 0; i < j; ++i)
//if(bstr < *(p->getKey(i)))
//break;
i = p->searchKey_less(bstr);
i = p->searchKey_less(_str, _len);
//insert existing key is ok, but not inserted in
//however, the tree-shape may change due to possible split in former code
bool ifexist = false;
if (i > 0 && bstr == *(p->getKey(i - 1)))
ifexist = true;
else
//if (i > 0 && bstr == *(p->getKey(i - 1)))
if (i > 0)
{
p->addKey(_key, i, true);
const Bstr* tmp = p->getKey(i-1);
int cmp_res = Util::compare(_str, _len, tmp->getStr(), tmp->getLen());
if(cmp_res == 0)
{
ifexist = true;
}
}
if(!ifexist)
{
p->addKey(_str, _len, i, true);
p->addValue(_val, i);
p->addNum();
request += _key->getLen();
request += _len;
p->setDirty();
this->TSM->updateHeap(p, p->getRank(), true);
}
this->TSM->request(request);
bstr.clear(); //NOTICE: must be cleared!
//bstr.clear(); //NOTICE: must be cleared!
return !ifexist; //QUERY(which case:return false)
}
bool
SITree::modify(const char* _str, unsigned _len, int _val)
SITree::modify(const char* _str, unsigned _len, unsigned _val)
{
if (_str == NULL || _len == 0)
{
printf("error in SITree-modify: empty string\n");
return false;
}
this->CopyToTransfer(_str, _len, 1);
//this->CopyToTransfer(_str, _len, 1);
this->request = 0;
const Bstr* _key = &transfer[1];
Bstr bstr = *_key;
//const Bstr* _key = &transfer[1];
//Bstr bstr = *_key;
int store;
SINode* ret = this->find(_key, &store, true);
if (ret == NULL || store == -1 || bstr != *(ret->getKey(store))) //tree is empty or not found
SINode* ret = this->find(_str, _len, &store, true);
if (ret == NULL || store == -1) //tree is empty or not found
{
bstr.clear();
//bstr.clear();
return false;
}
const Bstr* tmp = ret->getKey(store);
if (Util::compare(_str, _len, tmp->getStr(), tmp->getLen()) != 0) //tree is empty or not found
{
return false;
}
ret->setValue(_val, store);
ret->setDirty();
this->TSM->request(request);
bstr.clear();
//bstr.clear();
return true;
}
//this function is useful for search and modify, and range-query
SINode* //return the first key's position that >= *_key
SITree::find(const Bstr* _key, int* _store, bool ifmodify)
SITree::find(const char* _str, unsigned _len, int* _store, bool ifmodify)
{ //to assign value for this->bstr, function shouldn't be const!
if (this->root == NULL)
return NULL; //SITree Is Empty
SINode* p = root;
int i, j;
Bstr bstr = *_key; //local Bstr: multiple delete
//Bstr bstr = *_key; //local Bstr: multiple delete
while (!p->isLeaf())
{
if (ifmodify)
@ -277,7 +306,7 @@ SITree::find(const Bstr* _key, int* _store, bool ifmodify)
//for(i = 0; i < j; ++i) //BETTER(Binary-Search)
//if(bstr < *(p->getKey(i)))
//break;
i = p->searchKey_less(bstr);
i = p->searchKey_less(_str, _len);
p = p->getChild(i);
this->prepare(p);
@ -287,13 +316,15 @@ SITree::find(const Bstr* _key, int* _store, bool ifmodify)
//for(i = 0; i < j; ++i)
//if(bstr <= *(p->getKey(i)))
//break;
i = p->searchKey_lessEqual(bstr);
i = p->searchKey_lessEqual(_str, _len);
if (i == j)
*_store = -1; //Not Found
else
*_store = i;
bstr.clear();
//bstr.clear();
return p;
}
@ -312,24 +343,25 @@ SITree::remove(const char* _str, unsigned _len)
printf("error in SITree-remove: empty string\n");
return false;
}
this->CopyToTransfer(_str, _len, 1);
//this->CopyToTransfer(_str, _len, 1);
request = 0;
const Bstr* _key = &transfer[1];
//const Bstr* _key = &transfer[1];
SINode* ret;
if (this->root == NULL) //tree is empty
return false;
SINode* p = this->root;
SINode* q;
int i, j;
Bstr bstr = *_key;
//Bstr bstr = *_key;
while (!p->isLeaf())
{
j = p->getNum();
//for(i = 0; i < j; ++i)
//if(bstr < *(p->getKey(i)))
//break;
i = p->searchKey_less(bstr);
i = p->searchKey_less(_str, _len);
q = p->getChild(i);
this->prepare(q);
@ -343,6 +375,7 @@ SITree::remove(const char* _str, unsigned _len)
if (ret != NULL)
this->TSM->updateHeap(ret, 0, true);//non-sense node
this->TSM->updateHeap(q, q->getRank(), true);
if (q->isLeaf())
{
if (q->getPrev() == NULL)
@ -350,6 +383,7 @@ SITree::remove(const char* _str, unsigned _len)
if (q->getNext() == NULL)
this->leaves_tail = q;
}
if (p->getNum() == 0) //root shrinks
{
//this->leaves_head = q;
@ -365,7 +399,7 @@ SITree::remove(const char* _str, unsigned _len)
}
bool flag = false;
i = p->searchKey_equal(bstr);
i = p->searchKey_equal(_str, _len);
//WARN+NOTICE:here must check, because the key to remove maybe not exist
if (i != (int)p->getNum())
{
@ -386,7 +420,8 @@ SITree::remove(const char* _str, unsigned _len)
}
this->TSM->request(request);
bstr.clear();
//bstr.clear();
return flag; //i == j, not found
}
@ -419,12 +454,17 @@ SITree::release(SINode* _np) const
SITree::~SITree()
{
//cout << "SITree" << endl;
//cout << "delete TSM" << endl;
delete TSM;
TSM = NULL;
#ifdef DEBUG_KVSTORE
printf("already empty the buffer, now to delete all nodes in tree!\n");
//printf("already empty the buffer, now to delete all nodes in tree!\n");
#endif
//cout << "release" << endl;
//recursively delete each SINode
release(root);
//cout << "~SITree done" << endl;
}
void
@ -495,4 +535,5 @@ SITree::print(string s)
}
else;
#endif
}
}

28
KVstore/SITree/SITree.h
View File

@ -3,7 +3,7 @@
# Author: syzz
# Mail: 1181955272@qq.com
# Last Modified: 2015-04-26 16:44
# Description: struct and interface of the B+ tree
# Description: string2ID, including entity2id, literal2id, predicate2id
=============================================================================*/
#ifndef _KVSTORE_SITREE_SITREE_H
@ -21,7 +21,7 @@
class SITree
{
private:
unsigned int height; //0 indicates an empty tree
unsigned height; //0 indicates an empty tree
SINode* root;
SINode* leaves_head; //the head of LeafNode-list
SINode* leaves_tail; //the tail of LeafNode-list
@ -36,13 +36,19 @@ private:
//so lock is a must. Add lock to transfer is better than to add
//lock to every key/value. However, modify requires a lock for a
//key/value, and multiple search for different keys are ok!!!
Bstr transfer[3]; //0:transfer value searched; 1:copy key-data from const char*; 2:copy val-data from const char*
unsigned transfer_size[3];
//Bstr transfer[3]; //0:transfer value searched; 1:copy key-data from const char*; 2:copy val-data from const char*
//unsigned transfer_size[3];
//TODO: in all B+ trees, updat eoperation should lock the whole tree, while search operations not
//However, the transfer bstr maybe cause the parallism error!!!!
//Why we need the transfer? It is ok to pass the original string pointer to return
//A problem is that before the caller ends, the tree can not be modified(so a read-writ elock is required)
std::string storepath;
std::string filename; //ok for user to change
/* some private functions */
std::string getFilePath(); //in UNIX system
void CopyToTransfer(const char* _str, unsigned _len, unsigned _index);
//void CopyToTransfer(const char* _str, unsigned _len, unsigned _index);
void release(SINode* _np) const;
//tree's operations should be atom(if read nodes)
@ -54,14 +60,15 @@ private:
public:
SITree(); //always need to initial transfer
SITree(std::string _storepath, std::string _filename, std::string _mode, unsigned long long _buffer_size);
unsigned int getHeight() const;
unsigned getHeight() const;
void setHeight(unsigned _h);
SINode* getRoot() const;
//insert, search, remove, set
bool search(const char* _str, unsigned _len, int* _val);
bool insert(const char* _str, unsigned _len, int _val);
bool modify(const char* _str, unsigned _len, int _val);
bool search(const char* _str, unsigned _len, unsigned* _val);
bool insert(char* _str, unsigned _len, unsigned _val);
bool modify(const char* _str, unsigned _len, unsigned _val);
SINode* find(const Bstr* _key, int* store, bool ifmodify);
SINode* find(const char* _key, unsigned _len, int* store, bool ifmodify);
bool remove(const char* _str, unsigned _len);
bool save();
~SITree();
@ -71,4 +78,5 @@ public:
//(problem range between two extremes: not-modified, totally-modified)
//After saved, it's ok to continue operations on tree!
#endif
#endif

11
KVstore/SITree/heap/SIHeap.cpp
View File

@ -20,8 +20,8 @@ SIHeap::SIHeap(unsigned _size)
{
this->length = 0;
this->size = _size;
//this->heap = (Node**)malloc(this->size * sizeof(Node*)); //not use 4 or 8
this->heap = new SINode*[this->size];
this->heap = (SINode**)malloc(this->size * sizeof(SINode*)); //not use 4 or 8
//this->heap = new SINode*[this->size];
if (this->heap == NULL)
{
this->print("error in SIHeap: Allocation fail!");
@ -69,6 +69,7 @@ SIHeap::insert(SINode* _np)
{
if (this->length == this->size) //when full, reallocate
{
cout<<"check: double the heap"<<endl;
this->heap = (SINode**)realloc(this->heap, 2 * this->size * sizeof(SINode*));
if (this->heap == NULL)
{
@ -183,7 +184,8 @@ SIHeap::modify(SINode* _np, bool _flag) //control direction
SIHeap::~SIHeap()
{
delete[] this->heap;
//delete[] this->heap;
free(this->heap);
this->heap = NULL;
this->length = this->size = 0;
}
@ -193,4 +195,5 @@ SIHeap::print(string s)
{
#ifdef DEBUG_KVSTORE
#endif
}
}

3
KVstore/SITree/heap/SIHeap.h
View File

@ -38,4 +38,5 @@ public:
void print(std::string s); //DEBUG
};
#endif
#endif

9
KVstore/SITree/node/SIIntlNode.cpp
View File

@ -75,6 +75,7 @@ SIIntlNode::setChild(SINode* _child, int _index)
return false;
}
this->childs[_index] = _child;
return true;
}
@ -91,6 +92,7 @@ SIIntlNode::addChild(SINode* _child, int _index)
for (i = num; i >= _index; --i) //DEBUG: right bounder!!!
childs[i + 1] = childs[i];
childs[_index] = _child;
return true;
}
@ -106,6 +108,7 @@ SIIntlNode::subChild(int _index)
int i;
for (i = _index; i < num; ++i) //DEBUG: right bounder!!!
childs[i] = childs[i + 1];
return true;
}
@ -115,6 +118,7 @@ SIIntlNode::getSize() const
unsigned sum = INTL_SIZE, num = this->getNum(), i;
for (i = 0; i < num; ++i)
sum += keys[i].getLen();
return sum;
}
@ -140,6 +144,7 @@ SIIntlNode::split(SINode* _father, int _index)
_father->setDirty();
p->setDirty();
this->setDirty();
return p;
}
@ -235,6 +240,7 @@ SIIntlNode::coalesce(SINode* _father, int _index)
print("error in coalesce: Invalid case!");
//printf("error in coalesce: Invalid case!");
}
_father->setDirty();
p->setDirty();
this->setDirty();
@ -290,4 +296,5 @@ SIIntlNode::print(string s)
}
else;
#endif
}
}

3
KVstore/SITree/node/SIIntlNode.h
View File

@ -45,4 +45,5 @@ public:
*/
};
#endif
#endif

16
KVstore/SITree/node/SILeafNode.cpp
View File

@ -13,7 +13,7 @@ using namespace std;
void
SILeafNode::AllocValues()
{
values = new int[MAX_KEY_NUM];
values = new unsigned[MAX_KEY_NUM];
}
/*
@ -76,7 +76,7 @@ SILeafNode::getNext() const
return next;
}
int
unsigned
SILeafNode::getValue(int _index) const
{
int num = this->getNum();
@ -90,7 +90,7 @@ SILeafNode::getValue(int _index) const
}
bool
SILeafNode::setValue(int _val, int _index)
SILeafNode::setValue(unsigned _val, int _index)
{
int num = this->getNum();
if (_index < 0 || _index >= num)
@ -99,11 +99,12 @@ SILeafNode::setValue(int _val, int _index)
return false;
}
this->values[_index] = _val;
return true;
}
bool
SILeafNode::addValue(int _val, int _index)
SILeafNode::addValue(unsigned _val, int _index)
{
int num = this->getNum();
if (_index < 0 || _index > num)
@ -115,6 +116,7 @@ SILeafNode::addValue(int _val, int _index)
for (i = num - 1; i >= _index; --i)
this->values[i + 1] = this->values[i];
this->values[_index] = _val;
return true;
}
@ -130,6 +132,7 @@ SILeafNode::subValue(int _index)
int i;
for (i = _index; i < num - 1; ++i)
this->values[i] = this->values[i + 1];
return true;
}
@ -180,6 +183,7 @@ SILeafNode::split(SINode* _father, int _index)
_father->setDirty();
p->setDirty();
this->setDirty();
return p;
}
@ -275,6 +279,7 @@ SILeafNode::coalesce(SINode* _father, int _index)
print("error in coalesce: Invalid case!");
//printf("error in coalesce: Invalid case!");
}
_father->setDirty();
p->setDirty();
this->setDirty();
@ -362,4 +367,5 @@ SILeafNode::print(string s)
}
else;
#endif
}
}

17
KVstore/SITree/node/SILeafNode.h
View File

@ -16,7 +16,7 @@ class SILeafNode : public SINode
protected:
SINode* prev; //LeafNode
SINode* next;
int* values;
unsigned* values;
void AllocValues();
//void FreeValues();
public:
@ -27,18 +27,24 @@ public:
void Normal();
SINode* getPrev() const;
SINode* getNext() const;
int getValue(int _index) const;
bool setValue(int _val, int _index);
bool addValue(int _val, int _index);
unsigned getValue(int _index) const;
bool setValue(unsigned _val, int _index);
bool addValue(unsigned _val, int _index);
bool subValue(int _index);
void setPrev(SINode* _prev);
void setNext(SINode* _next);
unsigned getSize() const;
SINode* split(SINode* _father, int _index);
SINode* coalesce(SINode* _father, int _index);
void release();
~SILeafNode();
void print(std::string s); //DEBUG
/*non-sense virtual function
Node* getChild(int _index) const;
bool addChild(Node* _child, int _index);
@ -47,4 +53,5 @@ public:
};
//BETTER: prev isn't a must, and reverse-range can be achieved using recursive-next
#endif
#endif

79
KVstore/SITree/node/SINode.cpp
View File

@ -253,6 +253,28 @@ SINode::addKey(const Bstr* _key, int _index, bool ifcopy)
keys[_index].copy(_key);
else
keys[_index] = *_key;
return true;
}
bool
SINode::addKey(char* _str, unsigned _len, int _index, bool ifcopy)
{
int num = this->getNum();
if (_index < 0 || _index > num)
{
print(string("error in addKey: Invalid index ") + Util::int2string(_index));
return false;
}
int i;
//NOTICE: if num == MAX_KEY_NUM, will visit keys[MAX_KEY_NUM], not legal!!!
//however. tree operations ensure that: when node is full, not add but split first!
for (i = num - 1; i >= _index; --i)
keys[i + 1] = keys[i];
keys[_index].setStr(_str);
keys[_index].setLen(_len);
return true;
}
@ -270,6 +292,7 @@ SINode::subKey(int _index, bool ifdel)
keys[_index].release();
for (i = _index; i < num - 1; ++i)
keys[i] = keys[i + 1];
return true;
}
@ -296,6 +319,7 @@ SINode::searchKey_less(const Bstr& _bstr) const
low = mid + 1;
}
}
return low;
}
@ -327,4 +351,57 @@ SINode::searchKey_lessEqual(const Bstr& _bstr) const
return ret - 1;
else
return ret;
}
}
int
SINode::searchKey_less(const char* _str, unsigned _len) const
{
int num = this->getNum();
int low = 0, high = num - 1, mid = -1;
while (low <= high)
{
mid = (low + high) / 2;
//if (this->keys[mid] > _bstr)
if (Util::compare(this->keys[mid].getStr(), this->keys[mid].getLen(), _str, _len) > 0)
{
if (low == mid)
break;
high = mid;
}
else
{
low = mid + 1;
}
}
return low;
}
int
SINode::searchKey_equal(const char* _str, unsigned _len) const
{
int num = this->getNum();
//for(i = 0; i < num; ++i)
// if(bstr == *(p->getKey(i)))
// {
int ret = this->searchKey_less(_str, _len);
//if (ret > 0 && this->keys[ret - 1] == _bstr)
if (ret > 0 && Util::compare(this->keys[ret-1].getStr(), this->keys[ret-1].getLen(), _str, _len) == 0)
return ret - 1;
else
return num;
}
int
SINode::searchKey_lessEqual(const char* _str, unsigned _len) const
{
int ret = this->searchKey_less(_str, _len);
//if (ret > 0 && this->keys[ret - 1] == _bstr)
if (ret > 0 && Util::compare(this->keys[ret-1].getStr(), this->keys[ret-1].getLen(), _str, _len) == 0)
return ret - 1;
else
return ret;
}

22
KVstore/SITree/node/SINode.h
View File

@ -30,9 +30,9 @@ public:
static const unsigned NF_HT = 0xf00000; //height area in rank
static const unsigned NF_KN = 0x07f000; //NOTICE: decided by DEGREE
static const unsigned INTL_SIZE = sizeof(Bstr) * MAX_KEY_NUM;
static const unsigned LEAF_SIZE = sizeof(int) * MAX_KEY_NUM + INTL_SIZE;
int heapId;
static const unsigned LEAF_SIZE = sizeof(unsigned) * MAX_KEY_NUM + INTL_SIZE;
protected:
unsigned store; //store address, the BLock index
unsigned flag; //NF_RK, NF_IL,NF_ID, NF_IV, propety
@ -67,9 +67,11 @@ public:
void setStore(unsigned _store);
unsigned getFlag() const;
void setFlag(unsigned _flag);
const Bstr* getKey(int _index) const; //need to check the index
bool setKey(const Bstr* _key, int _index, bool ifcopy = false);
bool addKey(const Bstr* _key, int _index, bool ifcopy = false);
bool addKey(char* _str, unsigned _len, int _index, bool ifcopy = false);
bool subKey(int _index, bool ifdel = false);
//several binary key search utilities
@ -77,19 +79,26 @@ public:
int searchKey_equal(const Bstr& _bstr) const;
int searchKey_lessEqual(const Bstr& _bstr) const;
int searchKey_less(const char* _str, unsigned _len) const;
int searchKey_equal(const char* _str, unsigned _len) const;
int searchKey_lessEqual(const char* _str, unsigned _len) const;
//virtual functions: polymorphic
//NOTICE: not pure-virtual, not required to be implemented again, can be used now
virtual SINode* getChild(int _index) const { return NULL; };
virtual bool setChild(SINode* _child, int _index) { return true; };
virtual bool addChild(SINode* _child, int _index) { return true; };
virtual bool subChild(int _index) { return true; };
virtual SINode* getPrev() const { return NULL; };
virtual SINode* getNext() const { return NULL; };
virtual int getValue(int _index) const { return -1; };
virtual bool setValue(int _val, int _index) { return true; };
virtual bool addValue(int _val, int _index) { return true; };
virtual unsigned getValue(int _index) const { return -1; };
virtual bool setValue(unsigned _val, int _index) { return true; };
virtual bool addValue(unsigned _val, int _index) { return true; };
virtual bool subValue(int _index) { return true; };
virtual void setPrev(SINode* _prev) {};
virtual void setNext(SINode* _next) {};
//NOTICE: pure-virtual, must to be implemented again in the sub-class
virtual void Virtual() = 0;
virtual void Normal() = 0;
virtual unsigned getSize() const = 0; //return all memory owned
@ -114,4 +123,5 @@ public:
*to release the whole(pointer is invalid and rebuild problem)
*/
#endif
#endif

30
KVstore/SITree/storage/SIStorage.cpp
View File

@ -71,7 +71,8 @@ SIStorage::SIStorage(string& _filepath, string& _mode, unsigned* _height, unsign
else //_mode == "open"
{
//read basic information
int rootnum;
unsigned rootnum;
//int rootnum;
char c;
fread(this->treeheight, sizeof(unsigned), 1, this->treefp);
fread(&rootnum, sizeof(unsigned), 1, this->treefp);
@ -214,6 +215,7 @@ SIStorage::AllocBlock()
unsigned t = p->num;
this->freelist->next = p->next;
delete p;
return t;
}
@ -286,10 +288,11 @@ SIStorage::readNode(SINode* _np, long long* _request)
if (flag)
{
//to read all values
int tmp = -1;
unsigned tmp = INVALID;
//int tmp = -1;
for (i = 0; i < num; ++i)
{
fread(&tmp, sizeof(int), 1, treefp);
fread(&tmp, sizeof(unsigned), 1, treefp);
this->ReadAlign(&next);
_np->setValue(tmp, i);
}
@ -300,6 +303,7 @@ SIStorage::readNode(SINode* _np, long long* _request)
//_np->setMem();
this->updateHeap(_np, _np->getRank(), false);
bstr.clear();
return true;
}
@ -335,6 +339,7 @@ SIStorage::createNode(SINode*& _np) //cretae virtual nodes, not in-mem
_np->delDirty();
_np->delMem();
_np->setStore(Blocknum(ftell(treefp) - 4));
return true;
}
@ -343,6 +348,7 @@ SIStorage::writeNode(SINode* _np)
{
if (_np == NULL || !_np->inMem() || (_np->getRank() > 0 && !_np->isDirty()))
return false; //not need to write back
unsigned num = _np->getNum(), i;
bool flag = _np->isLeaf(), SpecialBlock = true;
/*
@ -392,12 +398,13 @@ SIStorage::writeNode(SINode* _np)
if (flag)
{
int tmp = -1;
//int tmp = -1;
unsigned tmp = INVALID;
//to write all values
for (i = 0; i < num; ++i)
{
tmp = _np->getValue(i);
fwrite(&tmp, sizeof(int), 1, treefp);
fwrite(&tmp, sizeof(unsigned), 1, treefp);
this->WriteAlign(&blocknum, SpecialBlock);
}
}
@ -408,6 +415,7 @@ SIStorage::writeNode(SINode* _np)
fwrite(&t, sizeof(unsigned), 1, treefp); //the end-block
//_np->setFlag(_np->getFlag() & ~Node::NF_ID);
_np->delDirty();
return true;
}
@ -419,7 +427,8 @@ SIStorage::readBstr(Bstr* _bp, unsigned* _next)
fread(&len, sizeof(unsigned), 1, this->treefp);
this->ReadAlign(_next);
//this->request(len);
char* s = (char*)malloc(len);
//char* s = (char*)malloc(len);
char* s = new char[len];
_bp->setLen(len);
for (i = 0; i + 4 < len; i += 4)
{
@ -437,6 +446,7 @@ SIStorage::readBstr(Bstr* _bp, unsigned* _next)
fseek(treefp, j, SEEK_CUR);
this->ReadAlign(_next);
_bp->setStr(s);
return true;
}
@ -462,6 +472,7 @@ SIStorage::writeBstr(const Bstr* _bp, unsigned* _curnum, bool& _SpecialBlock)
j = 4 - j;
fseek(treefp, j, SEEK_CUR);
this->WriteAlign(_curnum, _SpecialBlock);
return true;
}
@ -551,7 +562,10 @@ SIStorage::writeTree(SINode* _root) //write the whole tree back and close treefp
fputc(c & ~(1 << j), treefp);
bp = bp->next;
}
fflush(this->treefp);
//fclose(this->treefp);
return true;
}
@ -586,6 +600,7 @@ SIStorage::request(long long _needmem) //aligned to byte
return false;;
}
this->freemem -= _needmem;
return true;
}
@ -614,6 +629,7 @@ SIStorage::handler(unsigned long long _needmem) //>0
else
break;
}
return true;
}
@ -635,6 +651,7 @@ SIStorage::~SIStorage()
printf("already empty the freelist!\n");
#endif
delete this->minheap;
minheap = NULL;
#ifdef DEBUG_KVSTORE
printf("already empty the buffer heap!\n");
#endif
@ -657,3 +674,4 @@ SIStorage::print(string s)
fputs("\n", Util::debug_kvstore);
#endif
}

9
KVstore/SITree/storage/SIStorage.h
View File

@ -13,6 +13,14 @@
#include "../node/SILeafNode.h"
#include "../heap/SIHeap.h"
//TODO: whether to use heap or not, is a big question
//For single-query application, it seems that LRU list like VSTree is a better choice(no much cost in the buffer itself)
//But in multiple-queries case, things maybe different
//BETTER:
//add a heap position in node, to speed up the node-pointer searching
//lower the update times of heap, if the size is 128M, then each update is 27 at most
//if not update in time, then the heap maybe not be a heap, then why do we use heap? why not a simple array?
//It controls read, write, swap
class SIStorage
{
@ -70,3 +78,4 @@ public:
};
#endif

3
KVstore/Tree.h
View File

@ -1,4 +1,5 @@
//headers wrapper for all kinds of BPlusTree
#include "ISTree/ISTree.h"
#include "SITree/SITree.h"
#include "SITree/SITree.h"
#include "IVTree/IVTree.h"

119
Main/GMonitor.java Normal file
View File

@ -0,0 +1,119 @@
import java.io.*;
import java.net.*;
import java.net.URLEncoder;
import java.net.URLDecoder;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.Map;
public class GMonitor {
public static final String defaultServerIP = "127.0.0.1";
public static final int defaultServerPort = 9000;
//private Socket socket = null;
private String serverIP;
private int serverPort;
//private Socket socket = null;
public GMonitor() {
this.serverIP = GMonitor.defaultServerIP;
this.serverPort = GMonitor.defaultServerPort;
}
public GMonitor(int _port) {
this.serverIP = GMonitor.defaultServerIP;
this.serverPort = _port;
}
public GMonitor(String _ip, int _port) {
this.serverIP = _ip;
this.serverPort = _port;
}
public String sendGet(String param) {
String url = "http://" + this.serverIP + ":" + this.serverPort;
String result = "";
BufferedReader in = null;
System.out.println("parameter: "+param);
try {
param = URLEncoder.encode(param, "UTF-8");
}
catch (UnsupportedEncodingException ex) {
throw new RuntimeException("Broken VM does not support UTF-8");
}
try {
String urlNameString = url + "/" + param;
System.out.println("request: "+urlNameString);
URL realUrl = new URL(urlNameString);
// 打开和URL之间的连接
URLConnection connection = realUrl.openConnection();
// 设置通用的请求属性
connection.setRequestProperty("accept", "*/*");
connection.setRequestProperty("connection", "Keep-Alive");
connection.setRequestProperty("user-agent",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
// 建立实际的连接
connection.connect();
// 获取所有响应头字段
Map<String, List<String>> map = connection.getHeaderFields();
// 遍历所有的响应头字段
for (String key : map.keySet()) {
System.out.println(key + "--->" + map.get(key));
}
// 定义 BufferedReader输入流来读取URL的响应
in = new BufferedReader(new InputStreamReader(
connection.getInputStream()));
String line;
while ((line = in.readLine()) != null) {
result += line;
}
} catch (Exception e) {
System.out.println("error in get request: " + e);
e.printStackTrace();
}
// 使用finally块来关闭输入流
finally {
try {
if (in != null) {
in.close();
}
} catch (Exception e2) {
e2.printStackTrace();
}
}
return result;
}
public boolean connect() {
return true;
}
public boolean disconnect() {
return true;
}
public static void main(String[] args) {
if(args.length != 3)
{
System.err.println("Invalid parameter, the parammeters should be operation, ip/url and port!");
return;
}
String op = args[0];
String serverIP = args[1];
int serverPort = Integer.parseInt(args[2]);
// initialize the GStore server's IP address and port.
GMonitor gm = new GMonitor(serverIP, serverPort);
gm.connect();
String cmd = "?operation=" + op;
String msg = gm.sendGet(cmd);
gm.disconnect();
System.out.println(msg);
}
}

49
Main/gconsole.cpp
View File

@ -44,6 +44,9 @@ int query_handler(const vector<string>&);
int show_handler(const vector<string>&);
int add_handler(const vector<string>&);
int sub_handler(const vector<string>&);
//backup commands
int backup_handler(const vector<string>&);
int restore_handler(const vector<string>&);
//A structure which contains information on the commands this program can understand.
typedef struct {
@ -71,6 +74,8 @@ COMMAND native_commands[] = {
{ "restart", restart_handler, "\tRestart local server." },
{ "port", port_handler, "\tChange port of local server." },
{ "printport", printport_handler, "Print local server's port configuration."},
{ "backup", backup_handler, "\tBackup current database." },
{ "restore", restore_handler, "\tRestore a database backup." },
{ NULL, NULL, NULL }
};
@ -914,7 +919,7 @@ int drop_handler(const vector<string>& args) {
cerr << "You should use exactly the same db name as building, which should not end with \".db\"" << endl;
return -1;
}
database += ".db";
//database += ".db";
//remote mode
if (gc != NULL) {
@ -1112,7 +1117,7 @@ int add_handler(const vector<string>& args) {
cerr << "You should use exactly the same db name as building, which should not end with \".db\"" << endl;
return -1;
}
database += ".db";
//database += ".db";
Database _db(database);
if (!_db.insert(args[1])) {
cerr << "Failed to insert!" << endl;
@ -1135,8 +1140,9 @@ int sub_handler(const vector<string>& args) {
cerr << "You should use exactly the same db name as building, which should not end with \".db\"" << endl;
return -1;
}
database += ".db";
Database _db(database); if (!_db.remove(args[1])) {
//database += ".db";
Database _db(database);
if (!_db.remove(args[1])) {
cerr << "Failed to remove!" << endl;
return -1;
}
@ -1191,6 +1197,41 @@ int printport_handler(const vector<string>& args) {
return system("bin/gserver -P");
}
int backup_handler(const vector<string>& args) {
if (!args.empty()) {
cerr << "Too many arguments!" << endl;
return -1;
}
if (current_database == NULL) {
cerr << "No database loaded!" << endl;
return -1;
}
current_database->backup();
return 0;
}
int restore_handler(const vector<string>& args) {
if (args.size() != 1) {
cerr << "Exactly 1 argument required!" << endl;
return -1;
}
if (current_database != NULL) {
cerr << "Please unload your database first!" << endl;
return -1;
}
string database = args[0];
if (database.length() > 3 && database.substr(database.length() - 3, 3) == ".db") {
cerr << "You should use exactly the same db name as building, which should not end with \".db\"" << endl;
return -1;
}
//database += ".db";
Database db(database);
if (!db.restore()) {
return -1;
}
return 0;
}

1086
Main/ghttp.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

597
Main/gserver.cpp
View File

@ -1,264 +1,333 @@
/*=================================================================================
# Filename: gserver.cpp
# Author: Bookug Lobert
# Mail: 1181955272@qq.com
# Last Modified: 2016-02-26 19:15
# Description: first written by hanshuo, modified by zengli, improved by Wang Libo
=================================================================================*/
#include "../Server/Server.h"
#include "../Util/Util.h"
using namespace std;
#define GSERVER_PORT_FILE "bin/.gserver_port"
#define GSERVER_PORT_SWAP "bin/.gserver_port.swap"
#define GSERVER_LOG "logs/gserver.log"
bool isOnlyProcess(const char* argv0);
void checkSwap();
bool startServer();
bool stopServer();
int main(int argc, char* argv[])
{
//#ifdef DEBUG
Util util;
//#endif
string mode;
if (argc == 1) {
mode = "-h";
}
else {
mode = argv[1];
}
if (argc > 3 || (argc == 3 && mode != "-p" && mode != "--port")) {
cout << "Invalid arguments! Input \"bin/gserver -h\" for help." << endl;
return -1;
}
if (mode == "-h" || mode == "--help") {
cout << endl;
cout << "gStore Server (gServer)" << endl;
cout << endl;
cout << "Usage:\tbin/gserver [option]" << endl;
cout << endl;
cout << "Options:" << endl;
cout << "\t-h,--help\t\tDisplay this message." << endl;
cout << "\t-s,--start\t\tStart gServer." << endl;
cout << "\t-t,--stop\t\tStop gServer." << endl;
cout << "\t-r,--restart\t\tRestart gServer." << endl;
cout << "\t-p,--port [PORT=" << Socket::DEFAULT_CONNECT_PORT << "]\tChange connection port configuration, takes effect after restart if gServer running." << endl;
cout << "\t-P,--printport\t\tDisplay current connection port configuration." << endl;
cout << "\t-k,--kill\t\tKill existing gServer process(es), ONLY use when out of normal procedures." << endl;
cout << endl;
return 0;
}
if (mode == "-p" || mode == "--port") {
unsigned short port = Socket::DEFAULT_CONNECT_PORT;
if (argc == 3) {
if (!Util::isValidPort(string(argv[2]))) {
cout << "Invalid port: " << argv[2] << endl;
return -1;
}
else {
stringstream ss(argv[2]);
ss >> port;
}
}
if (!isOnlyProcess(argv[0])) {
ofstream out(GSERVER_PORT_SWAP, ios::out);
if (!out) {
cout << "Failed to change port!" << endl;
return -1;
}
out << port;
out.close();
cout << "Port will be changed to " << port << " after the current server stops or restarts." << endl;
return 0;
}
ofstream out(GSERVER_PORT_FILE, ios::out);
if (!out) {
cout << "Failed to change port!" << endl;
return -1;
}
out << port;
out.close();
cout << "Port changed to " << port << '.' << endl;
return 0;
}
if (mode == "-s" || mode == "--start") {
if (!isOnlyProcess(argv[0])) {
cout << "gServer already running!" << endl;
return -1;
}
if (startServer()) {
return 0;
}
else {
return -1;
}
}
if (mode == "-t" || mode == "--stop") {
if (isOnlyProcess(argv[0])) {
cout << "gServer not running!" << endl;
return -1;
}
if (stopServer()) {
return 0;
}
else {
return -1;
}
}
if (mode == "-r" || mode == "--restart") {
if (isOnlyProcess(argv[0])) {
cout << "gServer not running!" << endl;
return -1;
}
if (!stopServer()) {
return -1;
}
if (!startServer()) {
return -1;
}
return 0;
}
if (mode == "-P" || mode == "--printport") {
unsigned short port = Socket::DEFAULT_CONNECT_PORT;
ifstream in(GSERVER_PORT_FILE);
if (in) {
in >> port;
in.close();
}
cout << "Current connection port is " << port << '.' << endl;
unsigned short portSwap = 0;
ifstream inSwap(GSERVER_PORT_SWAP);
if (inSwap) {
inSwap >> portSwap;
inSwap.close();
}
if (portSwap != 0) {
cout << "Connection port to be changed to " << portSwap << '.' << endl;
}
return 0;
}
if (mode == "-k" || mode == "--kill") {
if (isOnlyProcess(argv[0])) {
cout << "No process to kill!" << endl;
return -1;
}
execl("/usr/bin/killall", "killall", Util::getExactPath(argv[0]).c_str(), NULL);
return 0;
}
cout << "Invalid arguments! Input \"bin/gserver -h\" for help." << endl;
return -1;
}
bool isOnlyProcess(const char* argv0) {
return Util::getSystemOutput("pidof " + Util::getExactPath(argv0)) == Util::int2string(getpid());
}
void checkSwap() {
if (access(GSERVER_PORT_SWAP, 00) != 0) {
return;
}
ifstream in(GSERVER_PORT_SWAP, ios::in);
if (!in) {
cout << "Failed in checkSwap(), port may not be changed." << endl;
return;
}
unsigned short port;
in >> port;
in.close();
ofstream out(GSERVER_PORT_FILE, ios::out);
if (!out) {
cout << "Failed in checkSwap(), port may not be changed." << endl;
return;
}
out << port;
out.close();
chmod(GSERVER_PORT_FILE, 0644);
string cmd = string("rm ") + GSERVER_PORT_SWAP;
system(cmd.c_str());
}
bool startServer() {
unsigned short port = Socket::DEFAULT_CONNECT_PORT;
ifstream in(GSERVER_PORT_FILE, ios::in);
if (!in) {
ofstream out(GSERVER_PORT_FILE, ios::out);
if (out) {
out << port;
out.close();
chmod(GSERVER_PORT_FILE, 0644);
}
}
else {
in >> port;
in.close();
}
pid_t fpid = fork();
// child
if (fpid == 0) {
if (!Util::dir_exist("logs")) {
Util::create_dir("logs");
}
freopen(GSERVER_LOG, "a", stdout);
freopen(GSERVER_LOG, "a", stderr);
Server server(port);
if (!server.createConnection()) {
cout << Util::getTimeString() << "Failed to create connection at port " << port << '.' << endl;
return false;
}
cout << Util::getTimeString() << "Server started at port " << port << '.' << endl;
server.listen();
exit(0);
return true;
}
// parent
else if (fpid > 0) {
cout << "Server started at port " << port << '.' << endl;
return true;
}
// fork failure
else {
cout << "Failed to start server at port " << port << '.' << endl;
return false;
}
}
bool stopServer() {
unsigned short port = Socket::DEFAULT_CONNECT_PORT;
ifstream in(GSERVER_PORT_FILE, ios::in);
if (in) {
in >> port;
in.close();
}
Socket socket;
if (!socket.create() || !socket.connect("127.0.0.1", port) || !socket.send("stop")) {
cout << "Failed to stop server at port " << port << '.' << endl;
return false;
}
string recv_msg;
socket.recv(recv_msg);
socket.close();
if (recv_msg != "server stopped.") {
cout << "Failed to stop server at port " << port << '.' << endl;
return false;
}
cout << "Server stopped at port " << port << '.' << endl;
checkSwap();
return true;
}
/*=================================================================================
# Filename: gserver.cpp
# Author: Bookug Lobert
# Mail: 1181955272@qq.com
# Last Modified: 2016-02-26 19:15
# Description: first written by hanshuo, modified by zengli, improved by Wang Libo
=================================================================================*/
#include "../Server/Server.h"
#include "../Util/Util.h"
using namespace std;
//#define GSERVER_PORT_FILE "bin/.gserver_port"
//#define GSERVER_PORT_SWAP "bin/.gserver_port.swap"
//#define GSERVER_LOG "logs/gserver.log"
bool isOnlyProcess(const char* argv0);
void checkSwap();
bool startServer(bool _debug);
bool stopServer();
int main(int argc, char* argv[])
{
//#ifdef DEBUG
Util util;
//#endif
string mode;
if (argc == 1) {
mode = "-h";
}
else {
mode = argv[1];
}
if (argc > 3 || (argc == 3 && mode != "-p" && mode != "--port")) {
cout << "Invalid arguments! Input \"bin/gserver -h\" for help." << endl;
return -1;
}
else if (mode == "-h" || mode == "--help") {
cout << endl;
cout << "gStore Server (gServer)" << endl;
cout << endl;
cout << "Usage:\tbin/gserver [option]" << endl;
cout << endl;
cout << "Options:" << endl;
cout << "\t-h,--help\t\tDisplay this message." << endl;
cout << "\t-s,--start\t\tStart gServer." << endl;
cout << "\t-t,--stop\t\tStop gServer." << endl;
cout << "\t-r,--restart\t\tRestart gServer." << endl;
cout << "\t-p,--port [PORT=" << Socket::DEFAULT_CONNECT_PORT << "]\tChange connection port configuration, takes effect after restart if gServer running." << endl;
cout << "\t-P,--printport\t\tDisplay current connection port configuration." << endl;
cout << "\t-d,--debug\t\tStart gServer in debug mode (keep gServer in the foreground)." << endl;
cout << "\t-k,--kill\t\tKill existing gServer process(es), ONLY use when out of normal procedures." << endl;
cout << endl;
return 0;
}
if (mode == "-p" || mode == "--port") {
unsigned short port = Socket::DEFAULT_CONNECT_PORT;
if (argc == 3) {
if (!Util::isValidPort(string(argv[2]))) {
cerr << "Invalid port: " << argv[2] << endl;
return -1;
}
else {
stringstream ss(argv[2]);
ss >> port;
}
}
if (!isOnlyProcess(argv[0])) {
//ofstream out(GSERVER_PORT_SWAP, ios::out);
ofstream out(Util::gserver_port_swap.c_str());
if (!out) {
cerr << "Failed to change port!" << endl;
return -1;
}
out << port;
out.close();
cout << "Port will be changed to " << port << " after the current server stops or restarts." << endl;
return 0;
}
//ofstream out(GSERVER_PORT_FILE, ios::out);
ofstream out(Util::gserver_port_file.c_str());
if (!out) {
cerr << "Failed to change port!" << endl;
return -1;
}
out << port;
out.close();
cout << "Port changed to " << port << '.' << endl;
return 0;
}
else if (mode == "-s" || mode == "--start") {
if (!isOnlyProcess(argv[0])) {
cerr << "gServer already running!" << endl;
return -1;
}
if (startServer(false)) {
sleep(1);
if (isOnlyProcess(argv[0])) {
cerr << "Server stopped unexpectedly. Check for port conflicts!" << endl;
return -1;
}
return 0;
}
else {
return -1;
}
}
else if (mode == "-t" || mode == "--stop") {
if (isOnlyProcess(argv[0])) {
cerr << "gServer not running!" << endl;
return -1;
}
if (stopServer()) {
return 0;
}
else {
return -1;
}
}
else if (mode == "-r" || mode == "--restart") {
if (isOnlyProcess(argv[0])) {
cerr << "gServer not running!" << endl;
return -1;
}
if (!stopServer()) {
return -1;
}
if (startServer(false)) {
sleep(1);
if (isOnlyProcess(argv[0])) {
cerr << "Server stopped unexpectedly. Check for port conflicts!" << endl;
return -1;
}
return 0;
}
else {
return -1;
}
return 0;
}
else if (mode == "-P" || mode == "--printport") {
unsigned short port = Socket::DEFAULT_CONNECT_PORT;
//ifstream in(GSERVER_PORT_FILE);
ifstream in(Util::gserver_port_file.c_str());
if (in) {
in >> port;
in.close();
}
cout << "Current connection port is " << port << '.' << endl;
unsigned short portSwap = 0;
//ifstream inSwap(GSERVER_PORT_SWAP);
ifstream inSwap(Util::gserver_port_swap.c_str());
if (inSwap) {
inSwap >> portSwap;
inSwap.close();
}
if (portSwap != 0) {
cout << "Connection port to be changed to " << portSwap << '.' << endl;
}
return 0;
}
else if (mode == "-k" || mode == "--kill") {
if (isOnlyProcess(argv[0])) {
cerr << "No process to kill!" << endl;
return -1;
}
execl("/usr/bin/killall", "killall", Util::getExactPath(argv[0]).c_str(), NULL);
return 0;
}
else {
cerr << "Invalid arguments! Type \"bin/gserver -h\" for help." << endl;
return -1;
}
}
bool isOnlyProcess(const char* argv0) {
return Util::getSystemOutput("pidof " + Util::getExactPath(argv0)) == Util::int2string(getpid());
}
void checkSwap() {
//if (access(GSERVER_PORT_SWAP, 00) != 0) {
if (access(Util::gserver_port_swap.c_str(), 00) != 0) {
return;
}
//ifstream in(GSERVER_PORT_SWAP, ios::in);
ifstream in(Util::gserver_port_swap.c_str());
if (!in) {
cerr << "Failed in checkSwap(), port may not be changed." << endl;
return;
}
unsigned short port;
in >> port;
in.close();
//ofstream out(GSERVER_PORT_FILE, ios::out);
ofstream out(Util::gserver_port_file.c_str());
if (!out) {
cerr << "Failed in checkSwap(), port may not be changed." << endl;
return;
}
out << port;
out.close();
//chmod(GSERVER_PORT_FILE, 0644);
chmod(Util::gserver_port_file.c_str(), 0644);
//string cmd = string("rm ") + GSERVER_PORT_SWAP;
string cmd = string("rm ") + Util::gserver_port_swap;
system(cmd.c_str());
}
bool startServer(bool _debug) {
unsigned short port = Socket::DEFAULT_CONNECT_PORT;
//ifstream in(GSERVER_PORT_FILE, ios::in);
ifstream in(Util::gserver_port_file.c_str());
if (!in) {
//ofstream out(GSERVER_PORT_FILE, ios::out);
ofstream out(Util::gserver_port_file.c_str());
if (out) {
out << port;
out.close();
//chmod(GSERVER_PORT_FILE, 0644);
chmod(Util::gserver_port_file.c_str(), 0644);
}
}
else {
in >> port;
in.close();
}
if (_debug) {
Server server(port);
if (!server.createConnection()) {
cerr << Util::getTimeString() << "Failed to create connection at port " << port << '.' << endl;
return false;
}
cout << Util::getTimeString() << "Server started at port " << port << '.' << endl;
server.listen();
return true;
}
pid_t fpid = fork();
// child
if (fpid == 0) {
if (!Util::dir_exist("logs")) {
Util::create_dir("logs");
}
freopen(Util::gserver_log.c_str(), "a", stdout);
freopen(Util::gserver_log.c_str(), "a", stderr);
int status;
while (true) {
fpid = fork();
// child, main process
if (fpid == 0) {
Server server(port);
if (!server.createConnection()) {
cerr << Util::getTimeString() << "Failed to create connection at port " << port << '.' << endl;
return false;
}
cout << Util::getTimeString() << "Server started at port " << port << '.' << endl;
server.listen();
exit(0);
return true;
}
// parent, deamon process
else if (fpid > 0) {
waitpid(fpid, &status, 0);
if (WIFEXITED(status)) {
exit(0);
return true;
}
cerr << Util::getTimeString() << "Server stopped abnormally, restarting server..." << endl;
}
// fork failure
else {
cerr << Util::getTimeString() << "Failed to start server: deamon fork failure." << endl;
return false;
}
}
}
// parent
else if (fpid > 0) {
cout << "Server started at port " << port << '.' << endl;
return true;
}
// fork failure
else {
cerr << "Failed to start server at port " << port << '.' << endl;
return false;
}
}
bool stopServer() {
unsigned short port = Socket::DEFAULT_CONNECT_PORT;
//ifstream in(GSERVER_PORT_FILE, ios::in);
ifstream in(Util::gserver_port_file.c_str());
if (in) {
in >> port;
in.close();
}
Socket socket;
if (!socket.create() || !socket.connect("127.0.0.1", port) || !socket.send("stop")) {
cerr << "Failed to stop server at port " << port << '.' << endl;
return false;
}
string recv_msg;
socket.recv(recv_msg);
socket.close();
if (recv_msg != "server stopped.") {
cerr << "Failed to stop server at port " << port << '.' << endl;
return false;
}
cout << "Server stopped at port " << port << '.' << endl;
checkSwap();
return true;
}

62
Main/gserver_backup_scheduler.cpp Normal file
View File

@ -0,0 +1,62 @@
#include "../Util/Util.h"
#include "../Server/Server.h"
using namespace std;
bool send_backup_msg(unsigned short _port, long _time);
int main(int argc, char* argv[]) {
//#ifdef DEBUG
Util util;
//#endif
if (argc != 3) {
return -1;
}
if (!Util::isValidPort(string(argv[1]))) {
return -1;
}
unsigned short port;
{
stringstream ss(argv[1]);
ss >> port;
}
long backup_time;
{
stringstream ss(argv[2]);
ss >> backup_time;
}
while (true) {
time_t cur_time = time(NULL);
while (cur_time >= backup_time) {
backup_time += Util::gserver_backup_interval;
}
sleep(backup_time - cur_time);
if (!send_backup_msg(port, backup_time)) {
return -1;
}
}
return 0;
}
bool send_backup_msg(unsigned short _port, long _time) {
stringstream ss;
ss << "backup " << _time;
Socket socket;
if (!socket.create() || !socket.connect("127.0.0.1", _port) || !socket.send(ss.str())) {
return false;
}
string recv_msg;
socket.recv(recv_msg);
socket.close();
if (recv_msg != "done") {
return false;
}
return true;
}

104
NOTES.md
View File

@ -1,3 +1,28 @@
git commit: http://www.cnblogs.com/ctaodream/p/6066694.html
<type>(<scope>) : <subject>
<空行>
<body>
<空行>
<footer>
type:
+ fix :修复bug
+ doc : 文档改变
+ style : 代码格式改变
+ refactor :某个已有功能重构
+ perf :性能优化
+ test :增加测试
+ build :改变了build工具 如 grunt换成了 npm
+ revert: 撤销上一次的 commit
scope:用来说明此次修改的影响范围 可以随便填写任何东西，commitizen也给出了几个 如：location 、browser、compile，不过我推荐使用
all ：表示影响面大 ，如修改了网络框架 会对真个程序产生影响
loation： 表示影响小，某个小小的功能
module：表示会影响某个模块 如登录模块、首页模块 、用户管理模块等等
subject: 用来简要描述本次改动，概述就好了
body:具体的修改信息 应该尽量详细
footer：放置写备注啥的，如果是 bug ，可以把bug id放入
---
# NOTICE
一般在实践应用中不是单机，而是CS模式，因为API也是基于CS模式的。
@ -7,6 +32,12 @@
在使用gserver时，不能在数据库没有unload时再用gbuild或其他命令修改数据库，仅限于C/S模式
将IRC聊天放到gstore文档上，freenode #gStore
storage中大量使用long类型，文件大小也可能达到64G，最好在64位机器上运行。
在将unsigned转换为long long或者unsigned long long的时候要注意补全问题，long类型不知是否类似情况
也许可以考虑Bstr中不存length只存str(内存中还有对齐的开销)，但对于特别长的串来说可能strlen过于耗时
# 推广
必须建立一个官方网站，可以展示下团队、demo，需要建立社区/论坛并维护
另外要有桌面应用或者网页应用，以可视化的方式操作数据库，类似virtuoso和neo4j那种
server 118.89.115.42 gstore-pku.com
@ -15,9 +46,27 @@ server 118.89.115.42 gstore-pku.com
1. gstore开发应用需要一个强大的中间层，设计范式
2. 组建configure，单机和分布式合并，以后直接在合并的基础上做修改
考虑使用hbase，结合云平台
---
论文：新的join策略，特殊的子图同态问题，如何选择顺序
动态估价的评估函数要考虑方向性，因为可能含literal变量，对应的候选集大小不可靠，只能单向。
但每条边总是含subject的，所以每条边总是可以备选的。不过问题是literal变量应该先做还是后做
另一种过滤方式：直接用key-value索引，比如?x-?y-constant，可能就比较适合，如果用vstree先过滤出的候选集太大的话
考虑对线状查询或星形查询做特殊处理，这里的形状仅指需要join的部分
比如?x-?y-constant, why not just use key-value to generate sequentially
但这种没有考虑到更远的约束，可能导致不少中间解是无效的，实际上线状图的拼接顺序也不一定是从两端开始
---
# 数据库可恢复性、事务与日志
以后可能要支持事务，那样就需要日志能够支持UNDO, REDO等操作，可以以后根据需要再修改。
数据库多版本，可以指定恢复到一个带标签的版本（比如时间作为标签，用户指定标签）
# 并行策略- 线程控制模块
不宜使用并行框架，可使用C的pthread，boost的thread库，或者启用C++11，gcc编译器需要高于4.8.1才能完整支持C++11
@ -75,21 +124,19 @@ http://blog.csdn.net/infoworld/article/details/8670951
# TODO
保证通过测试：shape和warp数据集，插删查操作都要测，bbug一系列特殊的查询也要跑
要在单机支持到10亿triple，最坏情况下最多有20亿entity和20亿literal，目前的编号方式是不行的(int扩展为unsigned)
最好在单机100G内存上支持起freebase(2.5B triples)这个规模的数据集，就像jena和virtuoso一样，慢不要紧
Http通讯中answer的\n\t问题，可能是因为没有url encode的关系
返回结果是否需要进行encode？如果是json是没有问题的
但若一个string中本身就含有空格或\t，还能正确传输么？
因为返回的结果不是URL，所以可以不处理
OPTIMIZE: in pre filter and only_pre_after_filter, p2s and p2o should be used instead of for-loop
In addition, pre with lower degree should be placed in early position
use hash-link for large candidates, use sort-link for small candidates and very large list
同时将ID的编码改为unsigned，无效标志-1改为最大值的宏, triple数目的类型也要改为unsigned
注意pre的ID还可以为-2，或者对于pre仍然用int，或者改函数的返回值为long long (还有一些没有用-1而是>=0)
triple num改为unsigned long long，争取单机最大支持到100亿数据集，只要entity等数目不超过20亿。
triple数目的类型应该为long long
---
将B+tree中叶节点的大的value分离出来，新建一套缓存，使用block机制，标记length为0表示未读取
类型bstr的length问题也需要解决
如果把类型直接改成long long，空间开销一下子就上升了一倍
解决方法：对于ID2string，仍然用char*和unsigned，但对于s2xx p2xx o2xx，应该用long long*和unsigned来表示，这样最高可支持到40亿triple
注意：在B+树中是以long long*的方式存，但读出后应该全部换成unsigned*和unsigned搭配的方式(最长支持20亿个po对)
UBSTR: 类型bstr的length问题也需要解决 如果把类型直接改成long long，空间开销一下子就上升了一倍
解决方法：对于ID2string，仍然用char*和unsigned，但对于s2xx p2xx o2xx，应该用unsigned long long*和unsigned来表示，这样最高可支持到40亿triple
(其实这个不是特别必要，很少会有这种情况，我们处理的triple数目一般限制在20亿，就算是type这种边，po对数也就是跟entity数目持平，很难达到5亿)
---
那么是否可以调整entity与literal的分界线，如果entity数目一般都比literal数目多的话
直接把literal从大到小编号，可在ID模块中指定顺序，这样每个Datbase模块应该有自己独特的分界线，其他模块用时也需要注意
@ -469,6 +516,8 @@ build db error if triple num > 500M
# BETTER
#### 添加数据访问层，数据范式和生成数据访问的源码
#### 在BasicQuery.cpp中的encodeBasicQuery函数中发现有pre_id==-1时就可以直接中止查询，返回空值！
#### 将KVstore模块中在堆中寻找Node*的操作改为用treap实现(或多存指针避开搜索？)
@ -519,6 +568,8 @@ http://www.oschina.net/question/188977_58777
# ADVICE
#### 考虑利用hdfs或者hbase，这样就可以利用各公司已有的数据库系统，但这是否会和已有的内外存交换冲突？
#### 数值型查询 实数域 [-bound, bound] 类型很难匹配，有必要单独编码么？ 数据集中不应有范围 Query中编码过滤后还需验证
x>a, x<b, >=, <=, a<x<b, x=c
vstree中遇到"1237"^^<...integer>时不直接取字符串，而是转换为数值并编码
@ -589,12 +640,16 @@ ACID? neo4j GraphDB
## 单个文件的gStore？嵌入式，轻便，类似sqlite，方便移植，做成库的方式给python等调用
## 联邦数据库，避免数据重导入，上层查询分块
mysql适合存属性，而gstore适合处理关系，trinity适合做算法
gstore是否能处理各种图算法需求呢，比如对两点求最短路？比如正则路径查询？
## 没必要关闭IO缓冲同步，因为用的基本都是C语言的输入输出操作
是否可以考虑用vf2算法来作子图同构？比较效率，相互结合？
考虑按谓词频度划分，比如建立两棵sp2o树，两者的缓存大小应该不同
gStore实现的是子图同态算法，要实现子图同构，只要保证中间表扩展时每一行中不会出现重复的元素即可
Consider the use of Bloom Filter and FM-sketches
---
@ -603,3 +658,28 @@ Consider the use of Bloom Filter and FM-sketches
http://www.hprd.org/download/
## GIT USAGE
http://www.ruanyifeng.com/blog/2014/06/git_remote.html
https://git-scm.com/book/zh/v1/%E8%B5%B7%E6%AD%A5-%E5%88%9D%E6%AC%A1%E8%BF%90%E8%A1%8C-Git-%E5%89%8D%E7%9A%84%E9%85%8D%E7%BD%AE
To see all operations in Git, use git reflog
#### how to commit a message
package.json
http://www.json.cn/
https://www.oschina.net/news/69705/git-commit-message-and-changelog-guide
https://sanwen8.cn/p/44eCof7.html
1. commit one by one, a commit just do one thing
2. place a empty line between head and body, body and footer
3. the first letter of header should be in uppercase, and the header should not be too long, just a wonderful summary
FIX: ... ADD:... REF:... 代码重构 SUB:...
4. each line should not be too long, add your real name and the influence in footer(maybe cause the code struct to change)

379
Parser/QueryParser.cpp
View File

@ -2,7 +2,7 @@
# Filename: QueryParser.cpp
# Author: Jiaqi, Chen
# Mail: chenjiaqi93@163.com
# Last Modified: 2016-07-14
# Last Modified: 2017-03-13
# Description: implement functions in QueryParser.h
=============================================================================*/
@ -10,29 +10,20 @@
using namespace std;
QueryParser::QueryParser()
{
_prefix_map.clear();
}
void QueryParser::SPARQLParse(const string &query, QueryTree &querytree)
{
//uncompress before use
dfa34_Table_uncompress();
pANTLR3_INPUT_STREAM input;
pSparqlLexer lex;
pANTLR3_COMMON_TOKEN_STREAM tokens;
pSparqlParser parser;
input = antlr3StringStreamNew((ANTLR3_UINT8 *)(query.c_str()), ANTLR3_ENC_UTF8, query.length(), (ANTLR3_UINT8 *)"QueryString");
pANTLR3_INPUT_STREAM input = antlr3StringStreamNew((ANTLR3_UINT8 *)(query.c_str()), ANTLR3_ENC_UTF8, query.length(), (ANTLR3_UINT8 *) "QueryString");
//input = antlr3FileStreamNew((pANTLR3_UINT8)filePath,ANTLR3_ENC_8BIT);
lex = SparqlLexerNew(input);
tokens = antlr3CommonTokenStreamSourceNew(ANTLR3_SIZE_HINT,TOKENSOURCE(lex));
parser = SparqlParserNew(tokens);
pSparqlLexer lex = SparqlLexerNew(input);
pANTLR3_COMMON_TOKEN_STREAM tokens = antlr3CommonTokenStreamSourceNew(ANTLR3_SIZE_HINT,TOKENSOURCE(lex));
pSparqlParser parser = SparqlParserNew(tokens);
SparqlParser_workload_return r = parser->workload(parser);
pANTLR3_BASE_TREE root = r.tree;
SparqlParser_workload_return workload_ret = parser->workload(parser);
pANTLR3_BASE_TREE root = workload_ret.tree;
if (printNode(root) > 0)
throw "[ERROR] Some errors are found in the SPARQL query request.";
@ -49,20 +40,20 @@ void QueryParser::SPARQLParse(const string &query, QueryTree &querytree)
int QueryParser::printNode(pANTLR3_BASE_TREE node, int dep)
{
const char* s = (const char*) node->getText(node)->chars;
const char *s = (const char *) node->getText(node)->chars;
ANTLR3_UINT32 treeType = node->getType(node);
int hasErrorNode = 0;
if (treeType == 0) hasErrorNode = 1;
int hasErrorNode = (treeType == 0 ? 1 : 0);
for (int i = 0; i < dep; i++) printf(" ");
printf("%d: %s\n",treeType,s);
for (int t = 0; t < dep; t++) printf("\t");
printf("%d: %s\n", treeType, s);
for (unsigned int i = 0; i < node->getChildCount(node); i++)
{
pANTLR3_BASE_TREE childNode = (pANTLR3_BASE_TREE) node->getChild(node, i);
hasErrorNode += printNode(childNode, dep + 1);
}
return hasErrorNode;
}
@ -77,9 +68,8 @@ void QueryParser::parseWorkload(pANTLR3_BASE_TREE node, QueryTree &querytree)
{
parseQuery(childNode, querytree);
}
else
//update 196
if (childNode->getType(childNode) == 196)
else if (childNode->getType(childNode) == 196)
{
parseUpdate(childNode, querytree);
}
@ -98,42 +88,41 @@ void QueryParser::parseQuery(pANTLR3_BASE_TREE node, QueryTree &querytree)
{
parsePrologue(childNode);
}
else
//select 155
if (childNode->getType(childNode) == 155)
else if (childNode->getType(childNode) == 155)
{
querytree.setQueryForm(QueryTree::Select_Query);
parseQuery(childNode, querytree);
}
else
//ask 13
if (childNode->getType(childNode) == 13)
else if (childNode->getType(childNode) == 13)
{
querytree.setQueryForm(QueryTree::Ask_Query);
querytree.setProjectionAsterisk();
parseQuery(childNode, querytree);
}
else
//select clause 156
if (childNode->getType(childNode) == 156)
else if (childNode->getType(childNode) == 156)
{
parseSelectClause(childNode, querytree);
}
else
//group graph pattern 77
if (childNode->getType(childNode) == 77)
else if (childNode->getType(childNode) == 77)
{
parseGroupPattern(childNode, querytree.getGroupPattern());
}
else
//group by 75
else if (childNode->getType(childNode) == 75)
{
parseGroupBy(childNode, querytree);
}
//order by 127
if (childNode->getType(childNode) == 127)
else if (childNode->getType(childNode) == 127)
{
parseOrderBy(childNode, querytree);
}
else
//offset 120 limit 102
if (childNode->getType(childNode) == 120 || childNode->getType(childNode) == 102)
else if (childNode->getType(childNode) == 120 || childNode->getType(childNode) == 102)
{
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 0);
@ -181,6 +170,7 @@ void QueryParser::parsePrefix(pANTLR3_BASE_TREE node)
for (unsigned int i = 0; i < node->getChildCount(node); i++)
{
pANTLR3_BASE_TREE childNode = (pANTLR3_BASE_TREE) node->getChild(node, i);
//prefix namespace 136
if (childNode->getType(childNode) == 136)
parseString(childNode, key, 0);
@ -189,7 +179,8 @@ void QueryParser::parsePrefix(pANTLR3_BASE_TREE node)
if (childNode->getType(childNode) == 89)
parseString(childNode, value, 0);
}
_prefix_map.insert(make_pair(key, value));
this->prefix_map[key] = value;
}
void QueryParser::replacePrefix(string &str)
@ -198,20 +189,21 @@ void QueryParser::replacePrefix(string &str)
{
int sep = str.find(":");
if (sep == -1) return;
string prefix = str.substr(0, sep + 1);
//blank node
if (prefix == "_:") return;
cout << "prefix = " << prefix << endl;
if (_prefix_map.find(prefix) != _prefix_map.end())
printf("prefix = %s\n", prefix.c_str());
if (this->prefix_map.count(prefix) != 0)
{
str = _prefix_map[prefix].substr(0, _prefix_map[prefix].length() - 1) + str.substr(sep + 1 ,str.length() - sep - 1) + ">";
cout << "str = " << str << endl;
str = this->prefix_map[prefix].substr(0, this->prefix_map[prefix].length() - 1) + str.substr(sep + 1 ,str.length() - sep - 1) + ">";
printf("str = %s\n",str.c_str());
}
else
{
cout << "prefix not found..." << endl;
printf("prefix not found...\n");
throw "[ERROR] Prefix is not found, please define it before use.";
}
}
@ -254,11 +246,10 @@ void QueryParser::parseSelectVar(pANTLR3_BASE_TREE node, QueryTree &querytree)
if (childNode->getType(childNode) == 200)
{
querytree.addProjectionVar();
QueryTree::ProjectionVar &var = querytree.getLastProjectionVar();
var.aggregate_type = QueryTree::ProjectionVar::None_type;
QueryTree::ProjectionVar &proj_var = querytree.getLastProjectionVar();
proj_var.aggregate_type = QueryTree::ProjectionVar::None_type;
parseString(childNode, var.var, 0);
querytree.addProjectionUsefulVar(var.var);
parseString(childNode, proj_var.var, 0);
}
}
}
@ -283,7 +274,7 @@ void QueryParser::parseSelectAggregateFunction(pANTLR3_BASE_TREE node, QueryTree
childNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 0);
//count 39
if (childNode->getType(childNode) != 39)
throw "[ERROR] The supported aggregate functions now is COUNT only.";
throw "[ERROR] The supported aggregate function now is COUNT only.";
bool distinct = false;
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 0);
@ -299,26 +290,25 @@ void QueryParser::parseSelectAggregateFunction(pANTLR3_BASE_TREE node, QueryTree
while (gchildNode->getType(gchildNode) == 190)
gchildNode = (pANTLR3_BASE_TREE) gchildNode->getChild(gchildNode, 0);
if (gchildNode->getType(gchildNode) != 200 && gchildNode->getType(gchildNode) != 14)
throw "[ERROR] The aggregate function COUNT can accepts only one var or *.";
throw "[ERROR] The aggregate function COUNT can accept only one var or *.";
querytree.addProjectionVar();
QueryTree::ProjectionVar &var = querytree.getLastProjectionVar();
var.aggregate_type = QueryTree::ProjectionVar::Count_type;
var.distinct = distinct;
QueryTree::ProjectionVar &proj_var = querytree.getLastProjectionVar();
proj_var.aggregate_type = QueryTree::ProjectionVar::Count_type;
proj_var.distinct = distinct;
if (gchildNode->getType(gchildNode) == 200)
{
parseString(gchildNode, var.aggregate_var, 0);
querytree.addProjectionUsefulVar(var.aggregate_var);
parseString(gchildNode, proj_var.aggregate_var, 0);
}
if (gchildNode->getType(gchildNode) == 14)
{
parseString(gchildNode, var.aggregate_var, 0); //for convenience, set aggregate_var *
parseString(gchildNode, proj_var.aggregate_var, 0); //for convenience, set aggregate_var *
querytree.setProjectionAsterisk();
}
childNode = (pANTLR3_BASE_TREE) node->getChild(node, 1);
parseString(childNode, var.var, 0);
parseString(childNode, proj_var.var, 0);
}
void QueryParser::parseGroupPattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern)
@ -354,6 +344,12 @@ void QueryParser::parseGroupPattern(pANTLR3_BASE_TREE node, QueryTree::GroupPatt
parseFilter(childNode, grouppattern);
}
//bind 17
if (childNode->getType(childNode) == 17)
{
parseBind(childNode, grouppattern);
}
//group graph pattern 77
//redundant {}
if (childNode->getType(childNode) == 77)
@ -368,7 +364,6 @@ void QueryParser::parsePattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &
printf("parsePattern\n");
string subject, predicate, object;
for (unsigned int i = 0; i < node->getChildCount(node); i++)
{
pANTLR3_BASE_TREE childNode = (pANTLR3_BASE_TREE) node->getChild(node, i);
@ -384,6 +379,7 @@ void QueryParser::parsePattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &
if (childNode->getType(childNode) == 142)
{
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 0);
//var 200
if (gchildNode->getType(gchildNode) == 200)
{
@ -407,18 +403,30 @@ void QueryParser::parsePattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &
if (i != 0 && i % 2 == 0) //triples same subject
{
grouppattern.addOnePattern(QueryTree::GroupPattern::Pattern( QueryTree::GroupPattern::Pattern::Element(subject),
QueryTree::GroupPattern::Pattern::Element(predicate),
QueryTree::GroupPattern::Pattern::Element(object)));
grouppattern.addOnePattern(QueryTree::GroupPattern::Pattern(QueryTree::GroupPattern::Pattern::Element(subject),
QueryTree::GroupPattern::Pattern::Element(predicate),
QueryTree::GroupPattern::Pattern::Element(object)));
//scope of filter
for (int j = (int)grouppattern.sub_grouppattern.size() - 1; j > 0; j--)
if (grouppattern.sub_grouppattern[j].type == QueryTree::GroupPattern::SubGroupPattern::Pattern_type &&
grouppattern.sub_grouppattern[j - 1].type == QueryTree::GroupPattern::SubGroupPattern::Filter_type)
{
QueryTree::GroupPattern::SubGroupPattern tmp(grouppattern.sub_grouppattern[j - 1]);
grouppattern.sub_grouppattern[j - 1] = grouppattern.sub_grouppattern[j];
grouppattern.sub_grouppattern[j] = tmp;
}
else break;
}
}
}
void QueryParser::parseOptionalOrMinus(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern)
{
//optional 124 minus 108
//optional 124
if (node->getType(node) == 124)
printf("parseOptional\n");
//minus 108
else if (node->getType(node) == 108)
printf("parseMinus\n");
@ -430,11 +438,11 @@ void QueryParser::parseOptionalOrMinus(pANTLR3_BASE_TREE node, QueryTree::GroupP
if (childNode->getType(childNode) == 77)
{
if (node->getType(node) == 124)
grouppattern.addOneOptionalOrMinus('o');
grouppattern.addOneOptional(QueryTree::GroupPattern::SubGroupPattern::Optional_type);
else if (node->getType(node) == 108)
grouppattern.addOneOptionalOrMinus('m');
grouppattern.addOneOptional(QueryTree::GroupPattern::SubGroupPattern::Minus_type);
parseGroupPattern(childNode, grouppattern.getLastOptionalOrMinus());
parseGroupPattern(childNode, grouppattern.getLastOptional());
}
}
}
@ -474,54 +482,58 @@ void QueryParser::parseFilter(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &g
while (childNode->getType(childNode) == 190)
childNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 0);
grouppattern.addOneFilterTree();
parseFilterTree(childNode, grouppattern, grouppattern.getLastFilterTree());
grouppattern.addOneFilter();
parseFilterTree(childNode, grouppattern.getLastFilter().root);
}
}
void QueryParser::parseFilterTree(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern, QueryTree::GroupPattern::FilterTreeNode &filter)
void QueryParser::parseFilterTree(pANTLR3_BASE_TREE node, QueryTree::GroupPattern::FilterTree::FilterTreeNode &filter)
{
printf("parseFilterTree\n");
switch (node->getType(node))
{
//! 192
case 192: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Not_type; break;
case 192: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Not_type; break;
//not 115
case 115: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Not_type; break;
case 115: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Not_type; break;
//or 125
case 125: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Or_type; break;
case 125: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Or_type; break;
//and 8
case 8: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::And_type; break;
case 8: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::And_type; break;
//equal 62
case 62: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Equal_type; break;
case 62: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Equal_type; break;
//not equal 116
case 116: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::NotEqual_type; break;
case 116: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::NotEqual_type; break;
//less 100
case 100: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Less_type; break;
case 100: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Less_type; break;
//less equal 101
case 101: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::LessOrEqual_type; break;
case 101: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::LessOrEqual_type; break;
//greater 72
case 72: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Greater_type; break;
case 72: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Greater_type; break;
//greater equal 73
case 73: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::GreaterOrEqual_type; break;
case 73: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::GreaterOrEqual_type; break;
//regex 150
case 150: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Builtin_regex_type; break;
case 150: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_regex_type; break;
//str 167
case 167: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Builtin_str_type; break;
case 167: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_str_type; break;
//isIRI 92
case 92: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_isiri_type; break;
//isURI 95
case 95: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_isuri_type; break;
//isLiteral 93
case 93: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_isliteral_type; break;
//isNumeric 94
case 94: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_isnumeric_type; break;
//lang 96
case 96: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Builtin_lang_type; break;
//langmatches 97
case 97: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Builtin_langmatches_type; break;
case 96: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_lang_type; break;
//langMatches 97
case 97: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_langmatches_type; break;
//bound 23
case 23: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Builtin_bound_type; break;
case 23: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_bound_type; break;
//in 81
case 81: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Builtin_in_type; break;
//exists 63
case 63: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Builtin_exists_type; break;
//not exists 117
case 117: filter.oper_type = QueryTree::GroupPattern::FilterTreeNode::Not_type; break;
case 81: filter.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_in_type; break;
default:
return;
@ -536,10 +548,10 @@ void QueryParser::parseFilterTree(pANTLR3_BASE_TREE node, QueryTree::GroupPatter
//in 81
if (childNode->getType(childNode) == 81)
{
filter.child.push_back(QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild());
filter.child[0].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type;
filter.child[0].node.oper_type = QueryTree::GroupPattern::FilterTreeNode::Builtin_in_type;
parseVarInExpressionList(node, grouppattern, filter.child[0].node);
filter.child.push_back(QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild());
filter.child[0].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::Tree_type;
filter.child[0].node.oper_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::Builtin_in_type;
parseVarInExpressionList(node, filter.child[0].node);
return;
}
@ -548,27 +560,7 @@ void QueryParser::parseFilterTree(pANTLR3_BASE_TREE node, QueryTree::GroupPatter
//in 81
if (node->getType(node) == 81)
{
parseVarInExpressionList(node, grouppattern, filter);
return;
}
//not exists 117
if (node->getType(node) == 117)
{
filter.child.push_back(QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild());
filter.child[0].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type;
filter.child[0].node.oper_type = QueryTree::GroupPattern::FilterTreeNode::Builtin_exists_type;
parseExistsGroupPattern(node, grouppattern, filter.child[0].node);
return;
}
//exists 63
if (node->getType(node) == 63)
{
parseExistsGroupPattern(node, grouppattern, filter);
parseVarInExpressionList(node, filter);
return;
}
@ -581,35 +573,36 @@ void QueryParser::parseFilterTree(pANTLR3_BASE_TREE node, QueryTree::GroupPatter
while (childNode->getType(childNode) == 190)
{
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 0);
if (gchildNode->getChildCount(gchildNode) != 0)
childNode = gchildNode;
else break;
}
filter.child.push_back(QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild());
filter.child.push_back(QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild());
//unary 190
if (childNode->getType(childNode) == 190)
{
filter.child[i].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type;
parseString(childNode, filter.child[i].arg, 1);
replacePrefix(filter.child[i].arg);
filter.child[i].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::String_type;
parseString(childNode, filter.child[i].str, 1);
replacePrefix(filter.child[i].str);
}
else if (childNode->getChildCount(childNode) == 0)
{
filter.child[i].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type;
parseString(childNode, filter.child[i].arg, 0);
replacePrefix(filter.child[i].arg);
filter.child[i].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::String_type;
parseString(childNode, filter.child[i].str, 0);
replacePrefix(filter.child[i].str);
}
else
{
filter.child[i].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type;
parseFilterTree(childNode, grouppattern, filter.child[i].node);
filter.child[i].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::Tree_type;
parseFilterTree(childNode, filter.child[i].node);
}
}
}
void QueryParser::parseVarInExpressionList(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern, QueryTree::GroupPattern::FilterTreeNode &filter)
void QueryParser::parseVarInExpressionList(pANTLR3_BASE_TREE node, QueryTree::GroupPattern::FilterTree::FilterTreeNode &filter)
{
printf("parseVarInExpressionList\n");
@ -628,31 +621,32 @@ void QueryParser::parseVarInExpressionList(pANTLR3_BASE_TREE node, QueryTree::Gr
while (childNode->getType(childNode) == 190)
{
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 0);
if (gchildNode->getChildCount(gchildNode) != 0)
childNode = gchildNode;
else break;
}
int last = filter.child.size();
filter.child.push_back(QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild());
filter.child.push_back(QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild());
//unary 190
if (childNode->getType(childNode) == 190)
{
filter.child[last].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type;
parseString(childNode, filter.child[last].arg, 1);
replacePrefix(filter.child[last].arg);
filter.child[last].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::String_type;
parseString(childNode, filter.child[last].str, 1);
replacePrefix(filter.child[last].str);
}
else if (childNode->getChildCount(childNode) == 0)
{
filter.child[last].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type;
parseString(childNode, filter.child[last].arg, 0);
replacePrefix(filter.child[last].arg);
filter.child[last].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::String_type;
parseString(childNode, filter.child[last].str, 0);
replacePrefix(filter.child[last].str);
}
else
{
filter.child[last].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type;
parseFilterTree(childNode, grouppattern, filter.child[last].node);
filter.child[last].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::Tree_type;
parseFilterTree(childNode, filter.child[last].node);
}
}
@ -667,51 +661,102 @@ void QueryParser::parseVarInExpressionList(pANTLR3_BASE_TREE node, QueryTree::Gr
while (gchildNode->getType(gchildNode) == 190)
{
pANTLR3_BASE_TREE ggchildNode = (pANTLR3_BASE_TREE) gchildNode->getChild(gchildNode, 0);
if (ggchildNode->getChildCount(ggchildNode) != 0)
gchildNode = ggchildNode;
else break;
}
int last = filter.child.size();
filter.child.push_back(QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild());
filter.child.push_back(QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild());
//unary 190
if (gchildNode->getType(gchildNode) == 190)
{
filter.child[last].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type;
parseString(gchildNode, filter.child[last].arg, 1);
replacePrefix(filter.child[last].arg);
filter.child[last].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::String_type;
parseString(gchildNode, filter.child[last].str, 1);
replacePrefix(filter.child[last].str);
}
else if (gchildNode->getChildCount(gchildNode) == 0)
{
filter.child[last].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type;
parseString(gchildNode, filter.child[last].arg, 0);
replacePrefix(filter.child[last].arg);
filter.child[last].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::String_type;
parseString(gchildNode, filter.child[last].str, 0);
replacePrefix(filter.child[last].str);
}
else
{
filter.child[last].node_type = QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type;
parseFilterTree(gchildNode, grouppattern, filter.child[last].node);
filter.child[last].node_type = QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild::Tree_type;
parseFilterTree(gchildNode, filter.child[last].node);
}
}
}
}
}
void QueryParser::parseExistsGroupPattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern, QueryTree::GroupPattern::FilterTreeNode &filter)
void QueryParser::parseBind(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern)
{
printf("parseExistsGroupPattern\n");
printf("parseBind\n");
string str, var;
pANTLR3_BASE_TREE childNode = (pANTLR3_BASE_TREE) node->getChild(node, 0);
//group graph pattern 77
if (childNode->getType(childNode) == 77)
//unary 190
while (childNode->getType(childNode) == 190)
{
grouppattern.addOneExistsGroupPattern();
filter.exists_grouppattern_id = (int)grouppattern.filter_exists_grouppatterns[(int)grouppattern.filter_exists_grouppatterns.size() - 1].size() - 1;
parseGroupPattern(childNode, grouppattern.getLastExistsGroupPattern());
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 0);
return;
if (gchildNode->getChildCount(gchildNode) != 0)
childNode = gchildNode;
else break;
}
//unary 190
if (childNode->getType(childNode) == 190)
{
parseString(childNode, str, 1);
replacePrefix(str);
}
else if (childNode->getChildCount(childNode) == 0)
{
parseString(childNode, str, 0);
replacePrefix(str);
}
else
{
throw "[ERROR] The BIND operator can't assign an expression to a var.";
}
childNode = (pANTLR3_BASE_TREE) node->getChild(node, 1);
//as 11
if (childNode->getType(childNode) == 11)
{
parseString(childNode, var, 1);
}
grouppattern.addOneBind();
grouppattern.getLastBind() = QueryTree::GroupPattern::Bind(str, var);
}
void QueryParser::parseGroupBy(pANTLR3_BASE_TREE node, QueryTree &querytree)
{
printf("parseGroupBy\n");
for (unsigned int i = 0; i < node->getChildCount(node); i++)
{
pANTLR3_BASE_TREE childNode = (pANTLR3_BASE_TREE) node->getChild(node, i);
//var 200
if (childNode->getType(childNode) == 200)
{
string var;
parseString(childNode, var, 0);
querytree.addGroupByVar(var);
}
else
throw "[ERROR] The supported GROUP BY key is var only.";
}
}
@ -728,9 +773,10 @@ void QueryParser::parseOrderBy(pANTLR3_BASE_TREE node, QueryTree &querytree)
{
string var;
bool desending = false;
for (unsigned int k = 0; k < childNode->getChildCount(childNode); k++)
for (unsigned int j = 0; j < childNode->getChildCount(childNode); j++)
{
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, k);
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, j);
//unary 190
while (gchildNode->getType(gchildNode) == 190)
@ -750,7 +796,7 @@ void QueryParser::parseOrderBy(pANTLR3_BASE_TREE node, QueryTree &querytree)
}
if (var.length() > 0)
querytree.addOrder(var, desending);
querytree.addOrderVar(var, desending);
}
}
}
@ -759,7 +805,7 @@ void QueryParser::parseString(pANTLR3_BASE_TREE node, string &str, int dep)
{
if (dep == 0)
{
str = (const char*) node->getText(node)->chars;
str = (const char *) node->getText(node)->chars;
return;
}
@ -789,7 +835,7 @@ void QueryParser::parseString(pANTLR3_BASE_TREE node, string &str, int dep)
//'''''' 172
//"""""" 173
string substr = (const char*) childNode->getText(childNode)->chars;
string substr = (const char *) childNode->getText(childNode)->chars;
if (childNode->getType(childNode) == 170)
substr = "\"" + substr.substr(1, substr.length() - 2) + "\"";
if (childNode->getType(childNode) == 172)
@ -827,9 +873,8 @@ void QueryParser::parseUpdate(pANTLR3_BASE_TREE node, QueryTree &querytree)
{
parsePrologue(childNode);
}
else
//insert 82
if (childNode->getType(childNode) == 82)
else if (childNode->getType(childNode) == 82)
{
//INSERT
//DATA
@ -838,13 +883,13 @@ void QueryParser::parseUpdate(pANTLR3_BASE_TREE node, QueryTree &querytree)
querytree.setUpdateType(QueryTree::Insert_Data);
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 1);
//triples template 186
if (gchildNode->getType(gchildNode) == 186)
parseTripleTemplate(gchildNode, querytree.getInsertPatterns());
}
else
//delete 48
if (childNode->getType(childNode) == 48 && childNode->getChildCount(childNode) > 0)
else if (childNode->getType(childNode) == 48 && childNode->getChildCount(childNode) > 0)
{
//DELETE
//DELETE
@ -852,6 +897,7 @@ void QueryParser::parseUpdate(pANTLR3_BASE_TREE node, QueryTree &querytree)
//TRIPLES_TEMPLATE
pANTLR3_BASE_TREE gchildNode = (pANTLR3_BASE_TREE) childNode->getChild(childNode, 0);
//data 41
if (gchildNode->getType(gchildNode) == 41)
{
@ -862,9 +908,8 @@ void QueryParser::parseUpdate(pANTLR3_BASE_TREE node, QueryTree &querytree)
if (gchildNode->getType(gchildNode) == 186)
parseTripleTemplate(gchildNode, querytree.getDeletePatterns());
}
else
//where 203
if (gchildNode->getType(gchildNode) == 203)
else if (gchildNode->getType(gchildNode) == 203)
{
querytree.setUpdateType(QueryTree::Delete_Where);
@ -877,9 +922,8 @@ void QueryParser::parseUpdate(pANTLR3_BASE_TREE node, QueryTree &querytree)
}
}
}
else
//modify 110
if (childNode->getType(childNode) == 110)
else if (childNode->getType(childNode) == 110)
{
parseModify(childNode, querytree);
}
@ -937,9 +981,9 @@ void QueryParser::parseTripleTemplate(pANTLR3_BASE_TREE node, QueryTree::GroupPa
if (j != 0 && j % 2 == 0) //triples same subject
{
grouppattern.addOnePattern(QueryTree::GroupPattern::Pattern( QueryTree::GroupPattern::Pattern::Element(subject),
QueryTree::GroupPattern::Pattern::Element(predicate),
QueryTree::GroupPattern::Pattern::Element(object)));
grouppattern.addOnePattern(QueryTree::GroupPattern::Pattern(QueryTree::GroupPattern::Pattern::Element(subject),
QueryTree::GroupPattern::Pattern::Element(predicate),
QueryTree::GroupPattern::Pattern::Element(object)));
}
}
}
@ -968,27 +1012,24 @@ void QueryParser::parseModify(pANTLR3_BASE_TREE node, QueryTree &querytree)
{
querytree.setUpdateType(QueryTree::Delete_Clause);
}
else
//insert 82
if (childNode->getType(childNode) == 82)
else if (childNode->getType(childNode) == 82)
{
if (querytree.getUpdateType() == QueryTree::Not_Update)
querytree.setUpdateType(QueryTree::Insert_Clause);
else if (querytree.getUpdateType() == QueryTree::Delete_Clause)
querytree.setUpdateType(QueryTree::Modify_Clause);
}
else
//triples template 186
if (childNode->getType(childNode) == 186)
else if (childNode->getType(childNode) == 186)
{
if (querytree.getUpdateType() == QueryTree::Delete_Clause)
parseTripleTemplate(childNode, querytree.getDeletePatterns());
else if (querytree.getUpdateType() == QueryTree::Insert_Clause || querytree.getUpdateType() == QueryTree::Modify_Clause)
parseTripleTemplate(childNode, querytree.getInsertPatterns());
}
else
//where 203
if (childNode->getType(childNode) == 203)
else if (childNode->getType(childNode) == 203)
{
//WHERE
//GROUP_GRAPH_PATTERN

60
Parser/QueryParser.h
View File

@ -2,7 +2,7 @@
# Filename: QueryParser.h
# Author: Jiaqi, Chen
# Mail: chenjiaqi93@163.com
# Last Modified: 2016-07-14
# Last Modified: 2017-03-13
# Description:
=============================================================================*/
@ -14,40 +14,40 @@
#include "SparqlParser.h"
#include "SparqlLexer.h"
class QueryParser{
private:
std::map<std::string,std::string> _prefix_map;
class QueryParser
{
public:
void SPARQLParse(const std::string &query, QueryTree &querytree);
int printNode(pANTLR3_BASE_TREE node,int dep = 0);
private:
std::map<std::string, std::string> prefix_map;
void parseWorkload(pANTLR3_BASE_TREE node, QueryTree &querytree);
int printNode(pANTLR3_BASE_TREE node, int dep = 0);
void parseQuery(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parsePrologue(pANTLR3_BASE_TREE node);
void parsePrefix(pANTLR3_BASE_TREE node);
void replacePrefix(std::string &str);
void parseSelectClause(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseSelectVar(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseSelectAggregateFunction(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseGroupPattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parsePattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseOptionalOrMinus(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseUnion(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseFilter(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseFilterTree(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern, QueryTree::GroupPattern::FilterTreeNode &filter);
void parseVarInExpressionList(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern, QueryTree::GroupPattern::FilterTreeNode &filter);
void parseExistsGroupPattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern, QueryTree::GroupPattern::FilterTreeNode &filter);
void parseOrderBy(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseString(pANTLR3_BASE_TREE node, std::string &str, int dep);
void parseWorkload(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseUpdate(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseTripleTemplate(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseModify(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseQuery(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parsePrologue(pANTLR3_BASE_TREE node);
void parsePrefix(pANTLR3_BASE_TREE node);
void replacePrefix(std::string &str);
void parseSelectClause(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseSelectVar(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseSelectAggregateFunction(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseGroupPattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parsePattern(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseOptionalOrMinus(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseUnion(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseFilter(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseFilterTree(pANTLR3_BASE_TREE node, QueryTree::GroupPattern::FilterTree::FilterTreeNode &filter);
void parseVarInExpressionList(pANTLR3_BASE_TREE node, QueryTree::GroupPattern::FilterTree::FilterTreeNode &filter);
void parseBind(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseGroupBy(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseOrderBy(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseString(pANTLR3_BASE_TREE node, std::string &str, int dep);
public:
QueryParser();
void SPARQLParse(const std::string &query, QueryTree &querytree);
void parseUpdate(pANTLR3_BASE_TREE node, QueryTree &querytree);
void parseTripleTemplate(pANTLR3_BASE_TREE node, QueryTree::GroupPattern &grouppattern);
void parseModify(pANTLR3_BASE_TREE node, QueryTree &querytree);
};
#endif /* QUERYPARSER_H_ */

87
Query/BasicQuery.cpp
View File

@ -102,7 +102,7 @@ BasicQuery::getVarName(int _var)
}
// get triples number, also sentences number
int
unsigned
BasicQuery::getTripleNum()
{
return this->triple_vt.size();
@ -121,7 +121,8 @@ int BasicQuery::getEdgeNeighborID(int _var, int _i_th_edge)
}
// get the ID of the i-th edge of _var
int BasicQuery::getEdgePreID(int _var, int _i_th_edge)
TYPE_PREDICATE_ID
BasicQuery::getEdgePreID(int _var, int _i_th_edge)
{
return this->edge_pre_id[_var][_i_th_edge];
}
@ -177,20 +178,20 @@ BasicQuery::getCandidateList(int _var)
return candidate_list[_var];
}
int
unsigned
BasicQuery::getCandidateSize(int _var)
{
return this->candidate_list[_var].size();
}
// get the result list of _var in the query graph
vector<int*>&
vector<unsigned*>&
BasicQuery::getResultList()
{
return result_list;
}
vector<int*>*
vector<unsigned*>*
BasicQuery::getResultListPointer()
{
return &result_list;
@ -339,21 +340,30 @@ BasicQuery::setReady(int _var)
}
void
BasicQuery::updateSubSig(int _sub_var_id, int _pre_id, int _obj_id, int _line_id, int _obj_var_id)
BasicQuery::updateSubSig(int _sub_var_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id, int _line_id, int _obj_var_id)
{
cout<<"sub var id: "<<_sub_var_id<<endl;
// update var(sub)_signature according this triple
//bool obj_is_str = (_obj_id == -1) && (_obj.at(0) != '?');
//if(obj_is_str)
if(_obj_id >= 0)
if(_obj_id != INVALID_ENTITY_LITERAL_ID)
//if(_obj_id >= 0)
{
//Signature::encodeStr2Entity(_obj.c_str(), this->var_sig[_sub_id]);
Signature::encodeStr2Entity(this->var_sig[_sub_var_id], _obj_id, Util::EDGE_OUT);
//Signature::encodeStr2Entity(this->var_sig[_sub_var_id], _obj_id, Util::EDGE_OUT);
if(_pre_id >= 0)
{
Signature::encodeEdge2Entity(this->var_sig[_sub_var_id], _pre_id, _obj_id, Util::EDGE_OUT);
}
else
{
Signature::encodeStr2Entity(this->var_sig[_sub_var_id], _obj_id, Util::EDGE_OUT);
}
}
if(_pre_id >= 0)
//DEBUG: if type of pre id is changed to usnigned, this will cause error
else if(_pre_id >= 0)
{
Signature::encodePredicate2Entity(this->var_sig[_sub_var_id], _pre_id, Util::EDGE_OUT);
Signature::encodePredicate2Entity(this->var_sig[_sub_var_id], _pre_id, Util::EDGE_OUT);
}
// update var(sub)_degree & edge_id according to this triple
@ -367,22 +377,30 @@ BasicQuery::updateSubSig(int _sub_var_id, int _pre_id, int _obj_id, int _line_id
}
void
BasicQuery::updateObjSig(int _obj_var_id, int _pre_id, int _sub_id, int _line_id, int _sub_var_id)
BasicQuery::updateObjSig(int _obj_var_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _sub_id, int _line_id, int _sub_var_id)
{
cout<<"obj var id: "<<_obj_var_id<<endl;
// update var(obj)_signature
//bool sub_is_str = (_sub_id == -1) && (_sub.at(0) != '?');
//if(sub_is_str)
if(_sub_id >= 0)
if(_sub_id != INVALID_ENTITY_LITERAL_ID)
//if(_sub_id >= 0)
{
//cout << "str2entity" << endl;
Signature::encodeStr2Entity(this->var_sig[_obj_var_id], _sub_id, Util::EDGE_IN);
//Signature::encodeStr2Entity(this->var_sig[_obj_var_id], _sub_id, Util::EDGE_IN);
if(_pre_id >= 0)
{
Signature::encodeEdge2Entity(this->var_sig[_obj_var_id], _pre_id, _sub_id, Util::EDGE_IN);
}
else
{
Signature::encodeStr2Entity(this->var_sig[_obj_var_id], _sub_id, Util::EDGE_IN);
}
}
if(_pre_id >= 0)
else if(_pre_id >= 0)
{
//cout << "pre2entity" << endl;
Signature::encodePredicate2Entity(this->var_sig[_obj_var_id], _pre_id, Util::EDGE_IN);
Signature::encodePredicate2Entity(this->var_sig[_obj_var_id], _pre_id, Util::EDGE_IN);
#ifdef DEBUG
//if(_obj_var_id == 1)
//{
@ -506,7 +524,8 @@ BasicQuery::encodeBasicQuery(KVstore* _p_kvstore, const vector<string>& _query_v
string& pre = this->triple_vt[i].predicate;
string& obj = this->triple_vt[i].object;
int pre_id = -1; //not found
//int pre_id = -1; //not found
TYPE_PREDICATE_ID pre_id = INVALID_PREDICATE_ID; //not found
if(pre[0] == '?') //pre var
{
pre_id = -2; //mark that this is a pre var
@ -547,11 +566,12 @@ BasicQuery::encodeBasicQuery(KVstore* _p_kvstore, const vector<string>& _query_v
bool sub_is_var = (sub_var_id != -1);
if(sub_is_var)
{
int obj_id = -1;
//int obj_id = -1;
TYPE_ENTITY_LITERAL_ID obj_id = INVALID_ENTITY_LITERAL_ID;
if(obj.at(0) != '?')
{
obj_id = _p_kvstore->getIDByEntity(obj);
if(obj_id == -1)
if(obj_id == INVALID_ENTITY_LITERAL_ID)
{
obj_id = _p_kvstore->getIDByLiteral(obj);
}
@ -572,7 +592,8 @@ BasicQuery::encodeBasicQuery(KVstore* _p_kvstore, const vector<string>& _query_v
bool obj_is_var = (obj_var_id != -1);
if(obj_is_var)
{
int sub_id = -1;
//int sub_id = -1;
TYPE_ENTITY_LITERAL_ID sub_id = INVALID_ENTITY_LITERAL_ID;
if(sub.at(0) != '?')
{
sub_id = _p_kvstore->getIDByEntity(sub);
@ -994,45 +1015,47 @@ BasicQuery::print(ostream& _out_stream)
// WARN:not used because this also considers the candidate not
// adding literals
int
int
BasicQuery::getVarID_MinCandidateList()
{
int min_var = -1;
int min_size = Util::TRIPLE_NUM_MAX;
unsigned min_size = Util::TRIPLE_NUM_MAX;
for(int i = 0; i < this->graph_var_num; i ++)
{
int tmp_size = (this->candidate_list[i]).size();
unsigned tmp_size = (this->candidate_list[i]).size();
if(tmp_size < min_size)
{
min_var = i;
min_size = tmp_size;
}
}
return min_var;
}
int
int
BasicQuery::getVarID_MaxCandidateList()
{
int max_var = -1;
int max_size = -1;
unsigned max_size = 0;
for(int i = 0; i < this->graph_var_num; i ++)
{
int tmp_size = (this->candidate_list[i]).size();
unsigned tmp_size = (this->candidate_list[i]).size();
if(tmp_size > max_size)
{
max_var = i;
max_size = tmp_size;
}
}
return max_var;
}
int
int
BasicQuery::getVarID_FirstProcessWhenJoin()
{
int min_var = -1;
int min_size = Util::TRIPLE_NUM_MAX;
unsigned min_size = Util::TRIPLE_NUM_MAX;
//int min_var2 = -1;
//int min_size2 = Util::TRIPLE_NUM_MAX;
for(int i = 0; i < this->graph_var_num; ++i)
@ -1049,7 +1072,7 @@ BasicQuery::getVarID_FirstProcessWhenJoin()
else
cout<<"var "<<i<<" is ready!"<<endl;
int tmp_size = (this->candidate_list[i]).size();
unsigned tmp_size = (this->candidate_list[i]).size();
//if(this->isLiteralVariable(i))
//{
//if(tmp_size < min_size2)
@ -1112,10 +1135,12 @@ string BasicQuery::triple_str()
stringstream _ss;
_ss<<"Triple num:"<<this->getTripleNum()<<endl;
for (int i=0;i<getTripleNum();i++)
for (int i = 0; i < getTripleNum(); i++)
{
_ss<<(this->getTriple(i).toString())<<endl;
}
return _ss.str();
}

20
Query/BasicQuery.h
View File

@ -120,7 +120,7 @@ private:
int retrieve_var_num;
string* var_name;
IDList* candidate_list;
vector<int*> result_list;
vector<unsigned*> result_list;
int* var_degree;
//whether has added the variable's literal candidate
@ -162,8 +162,8 @@ private:
//void updateSubSig(int _sub_id, int _pre_id, int _obj_id, std::string _obj, int _line_id);
//void updateObjSig(int _obj_id, int _pre_id, int _sub_id, std::string _sub, int _line_id);
void updateSubSig(int _sub_var_id, int _pre_id, int _obj_id, int _line_id, int _obj_var_id);
void updateObjSig(int _obj_var_id, int _pre_id, int _sub_id, int _line_id, int _sub_var_id);
void updateSubSig(int _sub_var_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id, int _line_id, int _obj_var_id);
void updateObjSig(int _obj_var_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _sub_id, int _line_id, int _sub_var_id);
//infos for predicate variables
vector<PreVar> pre_var;
@ -175,8 +175,8 @@ private:
map<int, int> selected_var_position;
public:
static const int MAX_VAR_NUM = 10;
static const int MAX_PRE_VAR_NUM = 10;
static const int MAX_VAR_NUM = 20;
static const int MAX_PRE_VAR_NUM = 20;
static const char NOT_JUST_SELECT = 'a';
static const char SELECT_VAR = 's';
@ -203,7 +203,7 @@ public:
int getIDByVarName(const string& _name);
// get triples number, also sentences number
int getTripleNum();
unsigned getTripleNum();
//check if a normal var is in select
bool isVarSelected(const std::string& _name) const;
@ -220,7 +220,7 @@ public:
int getEdgeNeighborID(int _var, int _i_th_edge);
// get the preID of the i-th edge of _var
int getEdgePreID(int _var, int _i_th_edge);
TYPE_PREDICATE_ID getEdgePreID(int _var, int _i_th_edge);
// get the type of the i-th edge of _var
char getEdgeType(int _var, int _i_th_edge);
@ -236,11 +236,11 @@ public:
// get the candidate list of _var in the query graph
IDList& getCandidateList(int _var);
int getCandidateSize(int _var);
unsigned getCandidateSize(int _var);
// get the result list of _var in the query graph
vector<int*>& getResultList();
vector<int*>* getResultListPointer();
vector<unsigned*>& getResultList();
vector<unsigned*>* getResultListPointer();
// get the entity signature of _var in the query graph
const EntityBitSet& getEntitySignature(int _var);

3163
Query/GeneralEvaluation.cpp
View File

File diff suppressed because it is too large Load Diff

134
Query/GeneralEvaluation.h
View File

@ -2,7 +2,7 @@
# Filename: GeneralEvaluation.h
# Author: Jiaqi, Chen
# Mail: chenjiaqi93@163.com
# Last Modified: 2016-09-12
# Last Modified: 2017-05-05
# Description:
=============================================================================*/
@ -22,7 +22,6 @@
#include "QueryTree.h"
#include "Varset.h"
#include "RegexExpression.h"
#include "ResultFilter.h"
#include "../Util/Triple.h"
class GeneralEvaluation
@ -30,35 +29,28 @@ class GeneralEvaluation
private:
QueryParser query_parser;
QueryTree query_tree;
SPARQLquery sparql_query;
std::vector <Varset> sparql_query_varset;
VSTree *vstree;
KVstore *kvstore;
TNUM* pre2num;
int limitID_predicate;
int limitID_literal;
StringIndex *stringindex;
Strategy strategy;
ResultFilter result_filter;
bool need_output_answer;
TYPE_TRIPLE_NUM *pre2num;
TYPE_PREDICATE_ID limitID_predicate;
TYPE_ENTITY_LITERAL_ID limitID_literal;
TYPE_ENTITY_LITERAL_ID limitID_entity;
public:
explicit GeneralEvaluation(VSTree *_vstree, KVstore *_kvstore, StringIndex *_stringindex, TNUM* _pre2num, int _limitID_predicate, int _limitID_literal):
vstree(_vstree), kvstore(_kvstore), stringindex(_stringindex), pre2num(_pre2num), limitID_predicate(_limitID_predicate), limitID_literal(_limitID_literal), need_output_answer(false)
GeneralEvaluation(VSTree *_vstree, KVstore *_kvstore, StringIndex *_stringindex, TYPE_TRIPLE_NUM *_pre2num, TYPE_PREDICATE_ID _limitID_predicate, TYPE_ENTITY_LITERAL_ID _limitID_literal,TYPE_ENTITY_LITERAL_ID _limitID_entity):
vstree(_vstree), kvstore(_kvstore), stringindex(_stringindex), pre2num(_pre2num), limitID_predicate(_limitID_predicate), limitID_literal(_limitID_literal), limitID_entity(_limitID_entity),temp_result(NULL)
{
}
std::vector<std::vector<std::string> > getSPARQLQueryVarset();
bool parseQuery(const std::string &_query);
QueryTree& getQueryTree();
bool doQuery();
void getBasicQuery(QueryTree::GroupPattern &grouppattern);
class FilterExistsGroupPatternResultSetRecord;
class FilterEvaluationMultitypeValue
{
public:
@ -141,32 +133,41 @@ class GeneralEvaluation
FilterEvaluationMultitypeValue():datatype(rdf_term), int_value(0), flt_value(0), dbl_value(0){}
};
class TempResult
{
public:
Varset var;
std::vector<int*> res;
class ResultPair
{
public:
unsigned *id;
std::vector<string> str;
ResultPair():id(NULL){}
};
Varset id_varset, str_varset;
std::vector<ResultPair> result;
Varset getAllVarset();
void release();
static int compareFunc(int *a, std::vector<int> &p, int *b, std::vector<int> &q);
void sort(int l, int r, std::vector<int> &p);
int findLeftBounder(std::vector<int> &p, int *b, std::vector<int> &q);
int findRightBounder(std::vector<int> &p, int *b, std::vector<int> &q);
static int compareRow(const ResultPair &x, const int x_id_cols, const std::vector<int> &x_pos,
const ResultPair &y, const int y_id_cols, const std::vector<int> &y_pos);
void sort(int l, int r, const std::vector<int> &this_pos);
int findLeftBounder(const std::vector<int> &this_pos, const ResultPair &x, const int x_id_cols, const std::vector<int> &x_pos) const;
int findRightBounder(const std::vector<int> &this_pos, const ResultPair &x, const int x_id_cols, const std::vector<int> &x_pos) const;
void convertId2Str(Varset convert_varset, StringIndex *stringindex, Varset &entity_literal_varset);
void doJoin(TempResult &x, TempResult &r);
void doUnion(TempResult &rt);
void doUnion(TempResult &r);
void doOptional(std::vector<bool> &binding, TempResult &x, TempResult &rn, TempResult &ra, bool add_no_binding);
void doMinus(TempResult &x, TempResult &r);
void doDistinct(TempResult &r);
void mapFilterTree2Varset(QueryTree::GroupPattern::FilterTreeNode &filter, Varset &v, Varset &entity_literal_varset);
void doFilter(QueryTree::GroupPattern::FilterTreeNode &filter, FilterExistsGroupPatternResultSetRecord &filter_exists_grouppattern_resultset_record, TempResult &r, StringIndex *stringindex, Varset &entity_literal_varset);
void getFilterString(QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild &child, FilterEvaluationMultitypeValue &femv, int *row, StringIndex *stringindex);
FilterEvaluationMultitypeValue matchFilterTree(QueryTree::GroupPattern::FilterTreeNode &filter, FilterExistsGroupPatternResultSetRecord &filter_exists_grouppattern_resultset_record, int *row, StringIndex *stringindex);
void getFilterString(QueryTree::GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild &child, FilterEvaluationMultitypeValue &femv, ResultPair &row, int id_cols, StringIndex *stringindex);
FilterEvaluationMultitypeValue matchFilterTree(QueryTree::GroupPattern::FilterTree::FilterTreeNode &filter, ResultPair &row, int id_cols, StringIndex *stringindex);
void doFilter(QueryTree::GroupPattern::FilterTree::FilterTreeNode &filter, TempResult &r, StringIndex *stringindex, Varset &entity_literal_varset);
void print();
void print(int no=-1);
};
class TempResultSet
@ -176,70 +177,41 @@ class GeneralEvaluation
void release();
int findCompatibleResult(Varset &_varset);
int findCompatibleResult(Varset &_id_varset, Varset &_str_varset);
void doJoin(TempResultSet &x, TempResultSet &r);
void doJoin(TempResultSet &x, TempResultSet &r, StringIndex *stringindex, Varset &entity_literal_varset);
void doUnion(TempResultSet &x, TempResultSet &r);
void doOptional(TempResultSet &x, TempResultSet &r);
void doMinus(TempResultSet &x, TempResultSet &r);
void doDistinct(Varset &projection, TempResultSet &r);
void doOptional(TempResultSet &x, TempResultSet &r, StringIndex *stringindex, Varset &entity_literal_varset);
void doMinus(TempResultSet &x, TempResultSet &r, StringIndex *stringindex, Varset &entity_literal_varset);
void doFilter(QueryTree::GroupPattern::FilterTree::FilterTreeNode &filter, TempResultSet &r, StringIndex *stringindex, Varset &entity_literal_varset);
void doFilter(QueryTree::GroupPattern::FilterTreeNode &filter, FilterExistsGroupPatternResultSetRecord &filter_exists_grouppattern_resultset_record, TempResultSet &r, StringIndex *stringindex, Varset &entity_literal_varset);
void doProjection1(Varset &proj, TempResultSet &r, StringIndex *stringindex, Varset &entity_literal_varset);
void doDistinct1(TempResultSet &r);
void print();
};
class EvaluationUnit
private:
TempResultSet *temp_result;
struct EvaluationStackStruct
{
private:
char type;
void *p;
public:
EvaluationUnit(char _type, void *_p = NULL):type(_type), p(_p){}
char getType()
{ return type; }
void *getPointer()
{ return p; }
QueryTree::GroupPattern grouppattern;
SPARQLquery *sparql_query;
vector<vector<string> > encode_varset;
};
std::vector<EvaluationStackStruct> rewriting_evaluation_stack;
std::vector<EvaluationUnit> semantic_evaluation_plan;
void generateEvaluationPlan(QueryTree::GroupPattern &grouppattern);
void dfsJoinableResultGraph(int x, vector < pair<char, int> > &node_info, vector < vector<int> > &edge, QueryTree::GroupPattern &grouppattern);
std::stack<TempResultSet*> semantic_evaluation_result_stack;
class FilterExistsGroupPatternResultSetRecord
{
public:
std::vector<TempResultSet*> resultset;
std::vector< std::vector<Varset> > common;
std::vector< std::vector< std::pair< std::vector<int>, std::vector<int> > > > common2resultset;
} filter_exists_grouppattern_resultset_record;
int countFilterExistsGroupPattern(QueryTree::GroupPattern::FilterTreeNode &filter);
void doEvaluationPlan();
class ExpansionEvaluationStackUnit
{
public:
ExpansionEvaluationStackUnit():result(NULL){}
QueryTree::GroupPattern grouppattern;
SPARQLquery sparql_query;
TempResultSet *result;
};
std::vector <ExpansionEvaluationStackUnit> expansion_evaluation_stack;
public:
TempResultSet* semanticBasedQueryEvaluation(QueryTree::GroupPattern &grouppattern);
bool expanseFirstOuterUnionGroupPattern(QueryTree::GroupPattern &grouppattern, std::deque<QueryTree::GroupPattern> &queue);
void queryRewriteEncodeRetrieveJoin(int dep);
TempResultSet* rewritingBasedQueryEvaluation(int dep);
bool needOutputAnswer();
void setNeedOutputAnswer();
void getFinalResult(ResultSet &ret_result);
void releaseResult();
void getFinalResult(ResultSet &result_str);
void releaseResultStack();
void prepareUpdateTriple(QueryTree::GroupPattern &update_pattern, TripleWithObjType *&update_triple, int &update_triple_num);
void prepareUpdateTriple(QueryTree::GroupPattern &update_pattern, TripleWithObjType *&update_triple, unsigned &update_triple_num);
};
#endif // _QUERY_GENERALEVALUATION_H

124
Query/IDList.cpp
View File

@ -16,38 +16,40 @@ IDList::IDList()
}
//return the _i-th id of the list if _i exceeds, return -1
int
IDList::getID(int _i)const
unsigned
IDList::getID(unsigned _i) const
{
if (this->size() > _i)
{
return this->id_list[_i];
}
return -1;
//return -1;
return INVALID;
}
bool
IDList::addID(int _id)
IDList::addID(unsigned _id)
{
//a check for duplicate case will be more reliable
this->id_list.push_back(_id);
return true;
}
int
IDList::size()const
unsigned
IDList::size() const
{
return this->id_list.size();
}
bool
IDList::empty()const
IDList::empty() const
{
return this->id_list.size() == 0;
}
bool
IDList::isExistID(int _id)const
IDList::isExistID(unsigned _id) const
{
// naive implementation of searching(linear search).
// you can use binary search when the id list is sorted, if necessary.
@ -62,15 +64,14 @@ IDList::isExistID(int _id)const
return false;
}
const vector<int>*
IDList::getList()const
const vector<unsigned>*
IDList::getList() const
{
return &(this->id_list);
}
int&
IDList::operator[](const int& _i)
unsigned&
IDList::operator[](const unsigned& _i)
{
if (this->size() > _i)
{
@ -105,7 +106,7 @@ IDList::clear()
}
void
IDList::copy(const vector<int>& _new_idlist)
IDList::copy(const vector<unsigned>& _new_idlist)
{
this->id_list = _new_idlist;
}
@ -116,8 +117,8 @@ IDList::copy(const IDList* _new_idlist)
this->id_list = *(_new_idlist->getList());
}
int
IDList::intersectList(const int* _id_list, int _list_len)
unsigned
IDList::intersectList(const unsigned* _id_list, unsigned _list_len)
{
if (_id_list == NULL || _list_len == 0)
{
@ -160,9 +161,9 @@ IDList::intersectList(const int* _id_list, int _list_len)
{
case 0:
{ //this bracket is needed if vars are defined in case
int id_i = 0;
int index_move_forward = 0;
vector<int>::iterator it = this->id_list.begin();
unsigned id_i = 0;
unsigned index_move_forward = 0;
vector<unsigned>::iterator it = this->id_list.begin();
while (it != (this->id_list).end())
{
int can_id = *it;
@ -186,16 +187,16 @@ IDList::intersectList(const int* _id_list, int _list_len)
it++;
}
remove_number = this->id_list.size() - index_move_forward;
vector<int>::iterator new_end = this->id_list.begin() + index_move_forward;
vector<unsigned>::iterator new_end = this->id_list.begin() + index_move_forward;
(this->id_list).erase(new_end, this->id_list.end());
break;
}
case 1:
{
vector<int> new_id_list;
for (int i = 0; i < _list_len; ++i)
vector<unsigned> new_id_list;
for (unsigned i = 0; i < _list_len; ++i)
{
if (Util::bsearch_vec_uporder(_id_list[i], this->getList()) != -1)
if (Util::bsearch_vec_uporder(_id_list[i], this->getList()) != INVALID)
new_id_list.push_back(_id_list[i]);
}
this->id_list = new_id_list;
@ -204,11 +205,11 @@ IDList::intersectList(const int* _id_list, int _list_len)
}
case 2:
{
vector<int> new_id_list;
int m = this->id_list.size(), i;
vector<unsigned> new_id_list;
unsigned m = this->id_list.size(), i;
for (i = 0; i < m; ++i)
{
if (Util::bsearch_int_uporder(this->id_list[i], _id_list, _list_len) != -1)
if (Util::bsearch_int_uporder(this->id_list[i], _id_list, _list_len) != INVALID)
new_id_list.push_back(this->id_list[i]);
}
this->id_list = new_id_list;
@ -223,25 +224,25 @@ IDList::intersectList(const int* _id_list, int _list_len)
return remove_number;
}
int
unsigned
IDList::intersectList(const IDList& _id_list)
{
// copy _id_list to the temp array first.
int temp_list_len = _id_list.size();
int* temp_list = new int[temp_list_len];
unsigned temp_list_len = _id_list.size();
unsigned* temp_list = new unsigned[temp_list_len];
//BETTER:not to copy, just achieve here
for (int i = 0; i < temp_list_len; i++)
for (unsigned i = 0; i < temp_list_len; i++)
{
temp_list[i] = _id_list.getID(i);
}
int remove_number = this->intersectList(temp_list, temp_list_len);
delete[]temp_list;
unsigned remove_number = this->intersectList(temp_list, temp_list_len);
delete[] temp_list;
return remove_number;
}
int
IDList::unionList(const int* _id_list, int _list_len, bool only_literal)
unsigned
IDList::unionList(const unsigned* _id_list, unsigned _list_len, bool only_literal)
{
if (_id_list == NULL || _list_len == 0)
return 0;
@ -249,23 +250,25 @@ IDList::unionList(const int* _id_list, int _list_len, bool only_literal)
if (only_literal)
{
//NOTICE:this means that the original is no literals and we need to add from a list(containing entities/literals)
int k = 0;
unsigned k = 0;
//NOTICE:literal id > entity id; the list is ordered
for (; k < _list_len; ++k)
if (Util::is_literal_ele(_id_list[k]))
break;
//TODO+BETTER: speed up the process to find the first literal
for (; k < _list_len; ++k)
this->addID(_id_list[k]);
return _list_len - k;
}
// O(n)
int origin_size = (this->id_list).size();
int* temp_list = new int[origin_size + _list_len];
int temp_list_len = 0;
unsigned origin_size = (this->id_list).size();
unsigned* temp_list = new unsigned[origin_size + _list_len];
unsigned temp_list_len = 0;
// union
{
int i = 0, j = 0;
unsigned i = 0, j = 0;
while (i < origin_size && j < _list_len)
{
if (this->id_list[i] == _id_list[j])
@ -298,7 +301,7 @@ IDList::unionList(const int* _id_list, int _list_len, bool only_literal)
}
}
int add_number = temp_list_len - origin_size;
unsigned add_number = temp_list_len - origin_size;
// update this IDList
this->clear();
@ -344,24 +347,24 @@ IDList::unionList(const int* _id_list, int _list_len, bool only_literal)
*/
}
int
unsigned
IDList::unionList(const IDList& _id_list, bool only_literal)
{
// copy _id_list to the temp array first.
int temp_list_len = _id_list.size();
int* temp_list = new int[temp_list_len];
unsigned temp_list_len = _id_list.size();
unsigned* temp_list = new unsigned[temp_list_len];
//BETTER:not to copy, just achieve here
for (int i = 0; i < temp_list_len; i++)
for (unsigned i = 0; i < temp_list_len; i++)
{
temp_list[i] = _id_list.getID(i);
}
int ret = this->unionList(temp_list, temp_list_len, only_literal);
unsigned ret = this->unionList(temp_list, temp_list_len, only_literal);
delete[] temp_list;
return ret;
}
IDList*
IDList::intersect(const IDList& _id_list, const int* _list, int _len)
IDList::intersect(const IDList& _id_list, const unsigned* _list, unsigned _len)
{
IDList* p = new IDList;
//if (_list == NULL || _len == 0) //just copy _id_list
@ -379,7 +382,7 @@ IDList::intersect(const IDList& _id_list, const int* _list, int _len)
//compare n(k+1) and nklogn: k0 = log(n/2)2 requiring that n>2
//k<=k0 binary search; k>k0 intersect
int method = -1; //0: intersect 1: search in vector 2: search in int*
int n = _id_list.size();
unsigned n = _id_list.size();
double k = 0;
if (n < _len)
{
@ -407,10 +410,10 @@ IDList::intersect(const IDList& _id_list, const int* _list, int _len)
case 0:
{ //this bracket is needed if vars are defined in case
int id_i = 0;
int num = _id_list.size();
for (int i = 0; i < num; ++i)
unsigned num = _id_list.size();
for (unsigned i = 0; i < num; ++i)
{
int can_id = _id_list.getID(i);
unsigned can_id = _id_list.getID(i);
while ((id_i < _len) && (_list[id_i] < can_id))
{
id_i++;
@ -431,20 +434,20 @@ IDList::intersect(const IDList& _id_list, const int* _list, int _len)
}
case 1:
{
for (int i = 0; i < _len; ++i)
for (unsigned i = 0; i < _len; ++i)
{
if (Util::bsearch_vec_uporder(_list[i], _id_list.getList()) != -1)
if (Util::bsearch_vec_uporder(_list[i], _id_list.getList()) != INVALID)
p->addID(_list[i]);
}
break;
}
case 2:
{
int m = _id_list.size(), i;
unsigned m = _id_list.size(), i;
for (i = 0; i < m; ++i)
{
int t = _id_list.getID(i);
if (Util::bsearch_int_uporder(t, _list, _len) != -1)
unsigned t = _id_list.getID(i);
if (Util::bsearch_int_uporder(t, _list, _len) != INVALID)
p->addID(t);
}
break;
@ -457,15 +460,16 @@ IDList::intersect(const IDList& _id_list, const int* _list, int _len)
return p;
}
int
IDList::erase(int i)
bool
IDList::erase(unsigned i)
{
id_list.erase(id_list.begin() + i, id_list.end());
return 0;
return true;
}
int
IDList::bsearch_uporder(int _key)
unsigned
IDList::bsearch_uporder(unsigned _key)
{
return Util::bsearch_vec_uporder(_key, this->getList());
}

30
Query/IDList.h
View File

@ -15,31 +15,31 @@ class IDList
{
public:
IDList();
int getID(int _i)const;
bool addID(int _id);
unsigned getID(unsigned _i) const;
bool addID(unsigned _id);
//check whether _id exists in this IDList.
bool isExistID(int _id) const;
int size() const;
bool isExistID(unsigned _id) const;
unsigned size() const;
bool empty() const;
const std::vector<int>* getList()const;
int& operator[] (const int & _i);
const std::vector<unsigned>* getList()const;
unsigned& operator[] (const unsigned & _i);
std::string to_str();
int sort();
void clear();
void copy(const std::vector<int>& _new_idlist);
void copy(const std::vector<unsigned>& _new_idlist);
void copy(const IDList* _new_idlist);
// intersect/union _id_list to this IDList, note that the two list must be ordered before using these two functions.
int intersectList(const int* _id_list, int _list_len);
int intersectList(const IDList&);
int unionList(const int* _id_list, int _list_len, bool only_literal=false);
int unionList(const IDList&, bool only_literal=false);
int bsearch_uporder(int _key);
static IDList* intersect(const IDList&, const int*, int);
unsigned intersectList(const unsigned* _id_list, unsigned _list_len);
unsigned intersectList(const IDList&);
unsigned unionList(const unsigned* _id_list, unsigned _list_len, bool only_literal=false);
unsigned unionList(const IDList&, bool only_literal=false);
unsigned bsearch_uporder(unsigned _key);
static IDList* intersect(const IDList&, const unsigned*, unsigned);
private:
std::vector<int> id_list;
int erase(int i);
std::vector<unsigned> id_list;
bool erase(unsigned i);
};
#endif //_QUERY_IDLIST_H

667
Query/QueryTree.cpp
View File

@ -2,7 +2,7 @@
# Filename: QueryTree.cpp
# Author: Jiaqi, Chen
# Mail: chenjiaqi93@163.com
# Last Modified: 2016-07-14
# Last Modified: 2017-03-13
# Description: implement functions in QueryTree.h
=============================================================================*/
@ -10,80 +10,126 @@
using namespace std;
void QueryTree::GroupPattern::FilterTreeNode::getVarset(Varset &varset)
void QueryTree::GroupPattern::FilterTree::FilterTreeNode::getVarset(Varset &varset)
{
for (int i = 0; i < (int)this->child.size(); i++)
{
if (this->child[i].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type && this->child[i].arg[0] == '?')
varset.addVar(this->child[i].arg);
if (this->child[i].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type)
if (this->child[i].node_type == FilterTreeChild::String_type && this->child[i].str[0] == '?')
varset.addVar(this->child[i].str);
if (this->child[i].node_type == FilterTreeChild::Tree_type)
this->child[i].node.getVarset(varset);
}
}
void QueryTree::GroupPattern::FilterTreeNode::print(vector<GroupPattern> &exist_grouppatterns, int dep)
void QueryTree::GroupPattern::FilterTree::FilterTreeNode::mapVarPos2Varset(Varset &varset, Varset &entity_literal_varset)
{
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Not_type) printf("!");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_regex_type) printf("REGEX");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_str_type) printf("STR");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_lang_type) printf("LANG");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_langmatches_type) printf("LANGMATCHES");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_bound_type) printf("BOUND");
if (this->oper_type == Not_type)
{
this->child[0].node.mapVarPos2Varset(varset, entity_literal_varset);
}
else if (this->oper_type == Or_type || this->oper_type == And_type)
{
this->child[0].node.mapVarPos2Varset(varset, entity_literal_varset);
this->child[1].node.mapVarPos2Varset(varset, entity_literal_varset);
}
else if (Equal_type <= this->oper_type && this->oper_type <= GreaterOrEqual_type)
{
if (this->child[0].node_type == FilterTreeChild::Tree_type)
this->child[0].node.mapVarPos2Varset(varset, entity_literal_varset);
else if (this->child[0].node_type == FilterTreeChild::String_type && this->child[0].str[0] == '?')
{
this->child[0].pos = Varset(this->child[0].str).mapTo(varset)[0];
this->child[0].isel = entity_literal_varset.findVar(this->child[0].str);
}
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_in_type)
if (this->child[1].node_type == FilterTreeChild::Tree_type)
this->child[1].node.mapVarPos2Varset(varset, entity_literal_varset);
else if (this->child[1].node_type == FilterTreeChild::String_type && this->child[1].str[0] == '?')
{
this->child[1].pos = Varset(this->child[1].str).mapTo(varset)[0];
this->child[1].isel = entity_literal_varset.findVar(this->child[1].str);
}
}
else if (this->oper_type == Builtin_regex_type ||
this->oper_type == Builtin_str_type ||
this->oper_type == Builtin_isiri_type ||
this->oper_type == Builtin_isuri_type ||
this->oper_type == Builtin_isliteral_type ||
this->oper_type == Builtin_isnumeric_type ||
this->oper_type == Builtin_lang_type ||
this->oper_type == Builtin_langmatches_type ||
this->oper_type == Builtin_bound_type ||
this->oper_type == Builtin_in_type)
{
if (this->child[0].node_type == FilterTreeChild::Tree_type)
this->child[0].node.mapVarPos2Varset(varset, entity_literal_varset);
else if (this->child[0].node_type == FilterTreeChild::String_type && this->child[0].str[0] == '?')
{
this->child[0].pos = Varset(this->child[0].str).mapTo(varset)[0];
this->child[0].isel = entity_literal_varset.findVar(this->child[0].str);
}
}
}
void QueryTree::GroupPattern::FilterTree::FilterTreeNode::print(int dep)
{
if (this->oper_type == Not_type) printf("!");
if (this->oper_type == Builtin_regex_type) printf("REGEX");
if (this->oper_type == Builtin_str_type) printf("STR");
if (this->oper_type == Builtin_isiri_type) printf("ISIRI");
if (this->oper_type == Builtin_isuri_type) printf("ISURI");
if (this->oper_type == Builtin_isliteral_type) printf("ISLITERAL");
if (this->oper_type == Builtin_isnumeric_type) printf("ISNUMERIC");
if (this->oper_type == Builtin_lang_type) printf("LANG");
if (this->oper_type == Builtin_langmatches_type) printf("LANGMATCHES");
if (this->oper_type == Builtin_bound_type) printf("BOUND");
if (this->oper_type == Builtin_in_type)
{
printf("(");
if (this->child[0].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type) printf("%s", this->child[0].arg.c_str());
if (this->child[0].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type) this->child[0].node.print(exist_grouppatterns, dep);
if (this->child[0].node_type == FilterTreeChild::String_type) printf("%s", this->child[0].str.c_str());
if (this->child[0].node_type == FilterTreeChild::Tree_type) this->child[0].node.print(dep);
printf(" IN (");
for (int i = 1; i < (int)this->child.size(); i++)
{
if (i != 1) printf(" , ");
if (this->child[i].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type) printf("%s", this->child[i].arg.c_str());
if (this->child[i].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type) this->child[i].node.print(exist_grouppatterns, dep);
if (i != 1) printf(", ");
if (this->child[i].node_type == FilterTreeChild::String_type) printf("%s", this->child[i].str.c_str());
if (this->child[i].node_type == FilterTreeChild::Tree_type) this->child[i].node.print(dep);
}
printf("))");
return;
}
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_exists_type)
{
printf("EXISTS");
exist_grouppatterns[this->exists_grouppattern_id].print(dep);
return;
}
printf("(");
if ((int)this->child.size() >= 1)
{
if (this->child[0].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type) printf("%s", this->child[0].arg.c_str());
if (this->child[0].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type) this->child[0].node.print(exist_grouppatterns, dep);
if (this->child[0].node_type == FilterTreeChild::String_type) printf("%s", this->child[0].str.c_str());
if (this->child[0].node_type == FilterTreeChild::Tree_type) this->child[0].node.print(dep);
}
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Or_type) printf(" || ");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::And_type) printf(" && ");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Equal_type) printf(" = ");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::NotEqual_type) printf(" != ");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Less_type) printf(" < ");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::LessOrEqual_type) printf(" <= ");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Greater_type) printf(" > ");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::GreaterOrEqual_type) printf(" >= ");
if (this->oper_type == Or_type) printf(" || ");
if (this->oper_type == And_type) printf(" && ");
if (this->oper_type == Equal_type) printf(" = ");
if (this->oper_type == NotEqual_type) printf(" != ");
if (this->oper_type == Less_type) printf(" < ");
if (this->oper_type == LessOrEqual_type) printf(" <= ");
if (this->oper_type == Greater_type) printf(" > ");
if (this->oper_type == GreaterOrEqual_type) printf(" >= ");
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_regex_type || this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_langmatches_type) printf(", ");
if (this->oper_type == Builtin_regex_type || this->oper_type == Builtin_langmatches_type) printf(", ");
if ((int)this->child.size() >= 2)
{
if (this->child[1].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type) printf("%s", this->child[1].arg.c_str());
if (this->child[1].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::Tree_type) this->child[1].node.print(exist_grouppatterns, dep);
if (this->child[1].node_type == FilterTreeChild::String_type) printf("%s", this->child[1].str.c_str());
if (this->child[1].node_type == FilterTreeChild::Tree_type) this->child[1].node.print(dep);
}
if ((int)this->child.size() >= 3)
{
if (this->oper_type == QueryTree::GroupPattern::FilterTreeNode::Builtin_regex_type && this->child[2].node_type == QueryTree::GroupPattern::FilterTreeNode::FilterTreeChild::String_type)
printf(", %s", this->child[2].arg.c_str());
if (this->oper_type == FilterTreeNode::Builtin_regex_type && this->child[2].node_type == FilterTreeChild::String_type)
printf(", %s", this->child[2].str.c_str());
}
printf(")");
@ -93,285 +139,280 @@ void QueryTree::GroupPattern::FilterTreeNode::print(vector<GroupPattern> &exist_
void QueryTree::GroupPattern::addOnePattern(Pattern _pattern)
{
this->patterns.push_back(_pattern);
this->sub_grouppattern.push_back(SubGroupPattern(SubGroupPattern::Pattern_type));
this->sub_grouppattern.back().pattern = _pattern;
}
void QueryTree::GroupPattern::addOneGroupUnion()
{
this->unions.push_back(GroupPatternUnions((int)this->patterns.size() - 1));
this->sub_grouppattern.push_back(SubGroupPattern(SubGroupPattern::Union_type));
}
void QueryTree::GroupPattern::addOneUnion()
{
int n = (int)this->unions.size();
this->unions[n - 1].grouppattern_vec.push_back(GroupPattern());
if (this->sub_grouppattern.back().type != SubGroupPattern::Union_type)
throw "QueryTree::GroupPattern::addOneUnion failed";
this->sub_grouppattern.back().unions.push_back(GroupPattern());
}
QueryTree::GroupPattern& QueryTree::GroupPattern::getLastUnion()
{
int n = (int)this->unions.size();
int m = (int)this->unions[n - 1].grouppattern_vec.size();
return this->unions[n - 1].grouppattern_vec[m - 1];
if (this->sub_grouppattern.back().type != SubGroupPattern::Union_type || this->sub_grouppattern.back().unions.empty())
throw "QueryTree::GroupPattern::getLastUnion failed";
return this->sub_grouppattern.back().unions.back();
}
void QueryTree::GroupPattern::addOneOptionalOrMinus(char _type)
void QueryTree::GroupPattern::addOneOptional(int _type)
{
this->optionals.push_back(OptionalOrMinusGroupPattern((int)this->patterns.size() - 1, (int)this->unions.size() - 1, _type));
SubGroupPattern::SubGroupPatternType type = (SubGroupPattern::SubGroupPatternType)_type;
if (type != SubGroupPattern::Optional_type && type != SubGroupPattern::Minus_type)
throw "QueryTree::GroupPattern::addOneOptional failed";
this->sub_grouppattern.push_back(SubGroupPattern(type));
}
QueryTree::GroupPattern& QueryTree::GroupPattern::getLastOptionalOrMinus()
QueryTree::GroupPattern& QueryTree::GroupPattern::getLastOptional()
{
int n = (int)this->optionals.size();
return this->optionals[n - 1].grouppattern;
if (this->sub_grouppattern.back().type != SubGroupPattern::Optional_type && this->sub_grouppattern.back().type != SubGroupPattern::Minus_type)
throw "QueryTree::GroupPattern::getLastOptional failed";
return this->sub_grouppattern.back().optional;
}
void QueryTree::GroupPattern::addOneFilterTree()
void QueryTree::GroupPattern::addOneFilter()
{
this->filters.push_back(FilterTreeRoot());
this->filter_exists_grouppatterns.push_back(vector<GroupPattern>());
this->sub_grouppattern.push_back(SubGroupPattern(SubGroupPattern::Filter_type));
}
QueryTree::GroupPattern::FilterTreeNode& QueryTree::GroupPattern::getLastFilterTree()
QueryTree::GroupPattern::FilterTree& QueryTree::GroupPattern::getLastFilter()
{
return this->filters[(int)(this->filters.size()) - 1].root;
if (this->sub_grouppattern.back().type != SubGroupPattern::Filter_type)
throw "QueryTree::GroupPattern::getLastFilter failed";
return this->sub_grouppattern.back().filter;
}
void QueryTree::GroupPattern::addOneExistsGroupPattern()
void QueryTree::GroupPattern::addOneBind()
{
int n = (int)this->filter_exists_grouppatterns.size();
this->filter_exists_grouppatterns[n - 1].push_back(GroupPattern());
this->sub_grouppattern.push_back(SubGroupPattern(SubGroupPattern::Bind_type));
}
QueryTree::GroupPattern& QueryTree::GroupPattern::getLastExistsGroupPattern()
QueryTree::GroupPattern::Bind& QueryTree::GroupPattern::getLastBind()
{
int n = (int)this->filter_exists_grouppatterns.size();
int m = (int)this->filter_exists_grouppatterns[n - 1].size();
return this->filter_exists_grouppatterns[n - 1][m - 1];
if (this->sub_grouppattern.back().type != SubGroupPattern::Bind_type)
throw "QueryTree::GroupPattern::getLastBind failed";
return this->sub_grouppattern.back().bind;
}
void QueryTree::GroupPattern::getVarset()
{
for (int i = 0; i < (int)this->patterns.size(); i++)
{
if (this->patterns[i].subject.value[0] == '?')
for (int i = 0; i < (int)this->sub_grouppattern.size(); i++)
if (this->sub_grouppattern[i].type == SubGroupPattern::Pattern_type)
{
this->patterns[i].varset.addVar(this->patterns[i].subject.value);
this->grouppattern_subject_object_maximal_varset.addVar(this->patterns[i].subject.value);
if (this->sub_grouppattern[i].pattern.subject.value[0] == '?')
{
this->sub_grouppattern[i].pattern.varset.addVar(this->sub_grouppattern[i].pattern.subject.value);
this->sub_grouppattern[i].pattern.subject_object_varset.addVar(this->sub_grouppattern[i].pattern.subject.value);
this->grouppattern_subject_object_maximal_varset.addVar(this->sub_grouppattern[i].pattern.subject.value);
}
if (this->sub_grouppattern[i].pattern.predicate.value[0] == '?')
{
this->sub_grouppattern[i].pattern.varset.addVar(this->sub_grouppattern[i].pattern.predicate.value);
this->grouppattern_predicate_maximal_varset.addVar(this->sub_grouppattern[i].pattern.predicate.value);
}
if (this->sub_grouppattern[i].pattern.object.value[0] == '?')
{
this->sub_grouppattern[i].pattern.varset.addVar(this->sub_grouppattern[i].pattern.object.value);
this->sub_grouppattern[i].pattern.subject_object_varset.addVar(this->sub_grouppattern[i].pattern.object.value);
this->grouppattern_subject_object_maximal_varset.addVar(this->sub_grouppattern[i].pattern.object.value);
}
this->grouppattern_resultset_minimal_varset += this->sub_grouppattern[i].pattern.varset;
this->grouppattern_resultset_maximal_varset += this->sub_grouppattern[i].pattern.varset;
}
if (this->patterns[i].predicate.value[0] == '?')
else if (this->sub_grouppattern[i].type == SubGroupPattern::Union_type)
{
this->patterns[i].varset.addVar(this->patterns[i].predicate.value);
this->grouppattern_predicate_maximal_varset.addVar(this->patterns[i].predicate.value);
}
if (this->patterns[i].object.value[0] == '?')
{
this->patterns[i].varset.addVar(this->patterns[i].object.value);
this->grouppattern_subject_object_maximal_varset.addVar(this->patterns[i].object.value);
}
this->grouppattern_resultset_minimal_varset = this->grouppattern_resultset_minimal_varset + this->patterns[i].varset;
this->grouppattern_resultset_maximal_varset = this->grouppattern_resultset_maximal_varset + this->patterns[i].varset;
}
Varset minimal_varset;
for (int i = 0; i < (int)this->unions.size(); i++)
{
Varset minimal_varset;
for (int j = 0; j < (int)this->unions[i].grouppattern_vec.size(); j++)
{
this->unions[i].grouppattern_vec[j].getVarset();
if (j == 0)
minimal_varset = minimal_varset + this->unions[i].grouppattern_vec[j].grouppattern_resultset_minimal_varset;
else
minimal_varset = minimal_varset * this->unions[i].grouppattern_vec[j].grouppattern_resultset_minimal_varset;
this->grouppattern_resultset_maximal_varset = this->grouppattern_resultset_maximal_varset + this->unions[i].grouppattern_vec[j].grouppattern_resultset_maximal_varset;
this->grouppattern_subject_object_maximal_varset = this->grouppattern_subject_object_maximal_varset + this->unions[i].grouppattern_vec[j].grouppattern_subject_object_maximal_varset;
this->grouppattern_predicate_maximal_varset = this->grouppattern_predicate_maximal_varset + this->unions[i].grouppattern_vec[j].grouppattern_predicate_maximal_varset;
for (int j = 0; j < (int)this->sub_grouppattern[i].unions.size(); j++)
{
this->sub_grouppattern[i].unions[j].getVarset();
if (j == 0)
minimal_varset = this->sub_grouppattern[i].unions[j].grouppattern_resultset_minimal_varset;
else
minimal_varset = minimal_varset * this->sub_grouppattern[i].unions[j].grouppattern_resultset_minimal_varset;
this->grouppattern_resultset_maximal_varset += this->sub_grouppattern[i].unions[j].grouppattern_resultset_maximal_varset;
this->grouppattern_subject_object_maximal_varset += this->sub_grouppattern[i].unions[j].grouppattern_subject_object_maximal_varset;
this->grouppattern_predicate_maximal_varset += this->sub_grouppattern[i].unions[j].grouppattern_predicate_maximal_varset;
}
this->grouppattern_resultset_minimal_varset += minimal_varset;
}
this->grouppattern_resultset_minimal_varset = this->grouppattern_resultset_minimal_varset + minimal_varset;
}
for (int i = 0; i < (int)this->optionals.size(); i++)
{
this->optionals[i].grouppattern.getVarset();
if (this->optionals[i].type == 'o')
else if (this->sub_grouppattern[i].type == SubGroupPattern::Optional_type)
{
this->grouppattern_resultset_maximal_varset = this->grouppattern_resultset_maximal_varset + this->optionals[i].grouppattern.grouppattern_resultset_maximal_varset;
this->grouppattern_subject_object_maximal_varset = this->grouppattern_subject_object_maximal_varset + this->optionals[i].grouppattern.grouppattern_subject_object_maximal_varset;
this->grouppattern_predicate_maximal_varset = this->grouppattern_predicate_maximal_varset + this->optionals[i].grouppattern.grouppattern_predicate_maximal_varset;
this->sub_grouppattern[i].optional.getVarset();
this->grouppattern_resultset_maximal_varset += this->sub_grouppattern[i].optional.grouppattern_resultset_maximal_varset;
this->grouppattern_subject_object_maximal_varset += this->sub_grouppattern[i].optional.grouppattern_subject_object_maximal_varset;
this->grouppattern_predicate_maximal_varset += this->sub_grouppattern[i].optional.grouppattern_predicate_maximal_varset;
}
}
for (int i = 0; i < (int)this->filters.size(); i++)
{
this->filters[i].root.getVarset(this->filters[i].varset);
}
for(int i = 0; i < (int)this->filter_exists_grouppatterns.size(); i++)
for (int j = 0; j < (int)this->filter_exists_grouppatterns[i].size(); j++)
else if (this->sub_grouppattern[i].type == SubGroupPattern::Minus_type)
{
this->filter_exists_grouppatterns[i][j].getVarset();
this->sub_grouppattern[i].optional.getVarset();
}
else if (this->sub_grouppattern[i].type == SubGroupPattern::Filter_type)
{
this->sub_grouppattern[i].filter.root.getVarset(this->sub_grouppattern[i].filter.varset);
}
else if (this->sub_grouppattern[i].type == SubGroupPattern::Bind_type)
{
this->sub_grouppattern[i].bind.varset = Varset(this->sub_grouppattern[i].bind.var);
this->grouppattern_resultset_minimal_varset += this->sub_grouppattern[i].bind.varset;
this->grouppattern_resultset_maximal_varset += this->sub_grouppattern[i].bind.varset;
}
}
bool QueryTree::GroupPattern::checkOnlyUnionOptionalFilterNoExists()
{
for (int i = 0; i < (int)this->unions.size(); i++)
{
for (int j = 0; j < (int)this->unions[i].grouppattern_vec.size(); j++)
if (!this->unions[i].grouppattern_vec[j].checkOnlyUnionOptionalFilterNoExists())
return false;
}
for (int i = 0; i < (int)this->optionals.size(); i++)
{
if (this->optionals[i].type != 'o')
return false;
if (!this->optionals[i].grouppattern.checkOnlyUnionOptionalFilterNoExists())
return false;
}
for (int i = 0; i < (int)this->filter_exists_grouppatterns.size(); i++)
if ((int)this->filter_exists_grouppatterns[i].size() != 0)
return false;
return true;
}
pair<Varset, Varset> QueryTree::GroupPattern::checkOptionalGroupPatternVarsAndSafeFilter(Varset occur , Varset ban, bool &check_condition)
pair<Varset, Varset> QueryTree::GroupPattern::checkNoMinusAndOptionalVarAndSafeFilter(Varset occur_varset, Varset ban_varset, bool &check_condition)
//return occur varset and ban varset
{
if (!check_condition) return make_pair(Varset(), Varset());
Varset this_ban;
Varset new_ban_varset;
int lastpattern = -1, lastunions = -1, lastoptional = -1;
while (check_condition && (lastpattern + 1 < (int)this->patterns.size() || lastunions + 1 < (int)this->unions.size() || lastoptional + 1 < (int)this->optionals.size()))
{
if (lastoptional + 1 < (int)this->optionals.size() && this->optionals[lastoptional + 1].lastpattern == lastpattern && this->optionals[lastoptional + 1].lastunions == lastunions)
//optional
for (int i = 0; i < (int)this->sub_grouppattern.size(); i++)
if (!check_condition) break;
else if (this->sub_grouppattern[i].type == SubGroupPattern::Pattern_type)
{
pair<Varset, Varset> sub_grouppattern_return_varset = this->optionals[lastoptional + 1].grouppattern.checkOptionalGroupPatternVarsAndSafeFilter(Varset(), ban, check_condition);
if (occur.hasCommonVar(sub_grouppattern_return_varset.second))
if (this->sub_grouppattern[i].pattern.varset.hasCommonVar(ban_varset))
check_condition = false;
Varset out = this->optionals[lastoptional + 1].grouppattern.grouppattern_resultset_maximal_varset - occur;
occur = occur + sub_grouppattern_return_varset.first;
this_ban = this_ban + sub_grouppattern_return_varset.second;
this_ban = this_ban + out;
ban = ban + this_ban;
lastoptional++;
occur_varset += this->sub_grouppattern[i].pattern.varset;
}
else if (lastunions + 1 < (int)this->unions.size() && this->unions[lastunions + 1].lastpattern == lastpattern)
//union
else if (this->sub_grouppattern[i].type == SubGroupPattern::Union_type)
{
Varset sub_grouppattern_occur, sub_grouppattern_ban;
Varset sub_occur_varset, sub_ban_varset;
for (int i = 0; i < (int)this->unions[lastunions + 1].grouppattern_vec.size(); i++)
for (int j = 0; j < (int)this->sub_grouppattern[i].unions.size(); j++)
{
pair<Varset, Varset> sub_grouppattern_result = this->unions[lastunions + 1].grouppattern_vec[i].checkOptionalGroupPatternVarsAndSafeFilter(occur, ban, check_condition);
pair<Varset, Varset> sub_return_varset =
this->sub_grouppattern[i].unions[j].checkNoMinusAndOptionalVarAndSafeFilter(occur_varset, ban_varset, check_condition);
if (i == 0)
sub_grouppattern_occur = sub_grouppattern_occur + sub_grouppattern_result.first;
if (j == 0)
sub_occur_varset = sub_return_varset.first;
else
sub_grouppattern_occur = sub_grouppattern_occur * sub_grouppattern_result.first;
sub_grouppattern_ban = sub_grouppattern_ban + sub_grouppattern_result.second;
sub_occur_varset = sub_occur_varset * sub_return_varset.first;
sub_ban_varset += sub_return_varset.second;
}
occur = occur + sub_grouppattern_occur;
this_ban = this_ban + sub_grouppattern_ban;
ban = ban + this_ban;
lastunions++;
new_ban_varset += sub_ban_varset;
occur_varset += sub_occur_varset;
ban_varset += new_ban_varset;
}
else
//triple pattern
else if (this->sub_grouppattern[i].type == SubGroupPattern::Optional_type)
{
if (this->patterns[lastpattern + 1].varset.hasCommonVar(ban))
pair<Varset, Varset> sub_return_varset =
this->sub_grouppattern[i].optional.checkNoMinusAndOptionalVarAndSafeFilter(Varset(), ban_varset, check_condition);
//occur before
if (occur_varset.hasCommonVar(sub_return_varset.second))
check_condition = false;
occur = occur + this->patterns[lastpattern + 1].varset;
lastpattern++;
new_ban_varset += sub_return_varset.second;
new_ban_varset += this->sub_grouppattern[i].optional.grouppattern_resultset_maximal_varset - occur_varset;
occur_varset += sub_return_varset.first;
ban_varset += new_ban_varset;
}
}
//filter
for (int i = 0; i < (int)this->filters.size(); i++)
if (!this->filters[i].varset.belongTo(occur))
{
check_condition = false;
break;
}
else if (this->sub_grouppattern[i].type == SubGroupPattern::Minus_type)
{
check_condition = false;
}
else if (this->sub_grouppattern[i].type == SubGroupPattern::Filter_type)
{
if (!this->sub_grouppattern[i].filter.varset.belongTo(occur_varset))
check_condition = false;
}
else if (this->sub_grouppattern[i].type == SubGroupPattern::Bind_type)
{
if (this->sub_grouppattern[i].bind.varset.hasCommonVar(ban_varset))
check_condition = false;
return make_pair(occur, this_ban);
occur_varset += this->sub_grouppattern[i].bind.varset;
}
return make_pair(occur_varset, new_ban_varset);
}
void QueryTree::GroupPattern::initPatternBlockid()
{
this->pattern_blockid.clear();
for (int i = 0; i < (int)this->patterns.size(); i++)
this->pattern_blockid.push_back(i);
for (int i = 0; i < (int)this->sub_grouppattern.size(); i++)
if (this->sub_grouppattern[i].type == SubGroupPattern::Pattern_type)
this->sub_grouppattern[i].pattern.blockid = i;
}
int QueryTree::GroupPattern::getRootPatternBlockID(int x)
{
if (this->pattern_blockid[x] == x) return x;
this->pattern_blockid[x] = getRootPatternBlockID(this->pattern_blockid[x]);
return this->pattern_blockid[x];
if (this->sub_grouppattern[x].type != SubGroupPattern::Pattern_type)
throw "QueryTree::GroupPattern::getRootPatternBlockID failed";
if (this->sub_grouppattern[x].pattern.blockid == x)
return x;
this->sub_grouppattern[x].pattern.blockid = this->getRootPatternBlockID(this->sub_grouppattern[x].pattern.blockid);
return this->sub_grouppattern[x].pattern.blockid;
}
void QueryTree::GroupPattern::mergePatternBlockID(int x, int y)
{
int px = getRootPatternBlockID(x);
int py = getRootPatternBlockID(y);
this->pattern_blockid[px] = py;
int px = this->getRootPatternBlockID(x);
int py = this->getRootPatternBlockID(y);
this->sub_grouppattern[px].pattern.blockid = py;
}
void QueryTree::GroupPattern::print(int dep)
{
for (int t = 0; t < dep; t++) printf("\t"); printf("{\n");
int lastpattern = -1, lastunions = -1, lastoptional = -1;
while (lastpattern + 1 < (int)this->patterns.size() || lastunions + 1 < (int)this->unions.size() || lastoptional + 1 < (int)this->optionals.size())
{
if (lastoptional + 1 < (int)this->optionals.size() && this->optionals[lastoptional + 1].lastpattern == lastpattern && this->optionals[lastoptional + 1].lastunions == lastunions)
//optional
for (int i = 0; i < (int)this->sub_grouppattern.size(); i++)
if (this->sub_grouppattern[i].type == SubGroupPattern::Pattern_type)
{
for (int t = 0; t <= dep; t++) printf("\t");
if (this->optionals[lastoptional + 1].type == 'o') printf("OPTIONAL\n");
if (this->optionals[lastoptional + 1].type == 'm') printf("MINUS\n");
this->optionals[lastoptional + 1].grouppattern.print(dep + 1);
lastoptional++;
printf("%s\t%s\t%s.\n", this->sub_grouppattern[i].pattern.subject.value.c_str(),
this->sub_grouppattern[i].pattern.predicate.value.c_str(),
this->sub_grouppattern[i].pattern.object.value.c_str());
}
else if (lastunions + 1 < (int)this->unions.size() && this->unions[lastunions + 1].lastpattern == lastpattern)
//union
else if (this->sub_grouppattern[i].type == SubGroupPattern::Union_type)
{
for (int i = 0; i < (int)this->unions[lastunions + 1].grouppattern_vec.size(); i++)
for (int j = 0; j < (int)this->sub_grouppattern[i].unions.size(); j++)
{
if (i != 0)
if (j != 0)
{
for (int t = 0; t <= dep; t++) printf("\t"); printf("UNION\n");
}
this->unions[lastunions + 1].grouppattern_vec[i].print(dep + 1);
this->sub_grouppattern[i].unions[j].print(dep + 1);
}
lastunions++;
}
else
//triple pattern
else if (this->sub_grouppattern[i].type == SubGroupPattern::Optional_type || this->sub_grouppattern[i].type == SubGroupPattern::Minus_type)
{
for (int t = 0; t <= dep; t++) printf("\t");
printf("%s\t%s\t%s.\n", this->patterns[lastpattern + 1].subject.value.c_str(), this->patterns[lastpattern + 1].predicate.value.c_str(), this->patterns[lastpattern + 1].object.value.c_str());
lastpattern++;
if (this->sub_grouppattern[i].type == SubGroupPattern::Optional_type) printf("OPTIONAL\n");
if (this->sub_grouppattern[i].type == SubGroupPattern::Minus_type) printf("MINUS\n");
this->sub_grouppattern[i].optional.print(dep + 1);
}
else if (this->sub_grouppattern[i].type == SubGroupPattern::Filter_type)
{
for (int t = 0; t <= dep; t++) printf("\t"); printf("FILTER\t");
this->sub_grouppattern[i].filter.root.print(dep + 1);
printf("\n");
}
else if (this->sub_grouppattern[i].type == SubGroupPattern::Bind_type)
{
for (int t = 0; t <= dep; t++) printf("\t");
printf("BIND(%s\tAS\t%s)", this->sub_grouppattern[i].bind.str.c_str(), this->sub_grouppattern[i].bind.var.c_str());
printf("\n");
}
}
//filter
for (int i = 0; i < (int)this->filters.size(); i++)
{
for (int t = 0; t <= dep; t++) printf("\t"); printf("FILTER\t");
this->filters[i].root.print(this->filter_exists_grouppatterns[i], dep + 1);
printf("\n");
}
for (int t = 0; t < dep; t++) printf("\t"); printf("}\n");
}
@ -390,7 +431,7 @@ QueryTree::QueryForm QueryTree::getQueryForm()
void QueryTree::setProjectionModifier(ProjectionModifier _projection_modifier)
{
projection_modifier = _projection_modifier;
this->projection_modifier = _projection_modifier;
}
QueryTree::ProjectionModifier QueryTree::getProjectionModifier()
@ -405,8 +446,7 @@ void QueryTree::addProjectionVar()
QueryTree::ProjectionVar& QueryTree::getLastProjectionVar()
{
int n = (int)this->projection.size();
return this->projection[n - 1];
return this->projection.back();
}
vector<QueryTree::ProjectionVar>& QueryTree::getProjection()
@ -414,24 +454,27 @@ vector<QueryTree::ProjectionVar>& QueryTree::getProjection()
return this->projection;
}
vector<string> QueryTree::getProjectionVar()
Varset QueryTree::getProjectionVarset()
{
vector<string> var_vec;
Varset varset;
for (int i = 0; i < (int)this->projection.size(); i++)
var_vec.push_back(this->projection[i].var);
varset.addVar(this->projection[i].var);
return var_vec;
return varset;
}
void QueryTree::addProjectionUsefulVar(string &_var)
Varset QueryTree::getResultProjectionVarset()
{
this->projection_useful_varset.addVar(_var);
}
Varset varset;
Varset& QueryTree::getProjectionUsefulVar()
{
return this->projection_useful_varset;
for (int i = 0; i < (int)this->projection.size(); i++)
if (this->projection[i].aggregate_type == ProjectionVar::None_type)
varset.addVar(this->projection[i].var);
else if (this->projection[i].aggregate_var != "*")
varset.addVar(this->projection[i].aggregate_var);
return varset;
}
void QueryTree::setProjectionAsterisk()
@ -444,41 +487,34 @@ bool QueryTree::checkProjectionAsterisk()
return this->projection_asterisk;
}
bool QueryTree::checkSelectCompatibility()
void QueryTree::addGroupByVar(string &_var)
{
bool all_var = true, all_aggregate_function = true;
if (this->checkProjectionAsterisk())
return true;
for (int i = 0; i < (int)this->projection.size(); i++)
{
if (this->projection[i].aggregate_type != ProjectionVar::None_type)
all_var = false;
if (this->projection[i].aggregate_type == ProjectionVar::None_type)
all_aggregate_function = false;
}
return all_var || all_aggregate_function;
this->group_by.addVar(_var);
}
bool QueryTree::atLeastOneAggregateFunction()
Varset& QueryTree::getGroupByVarset()
{
for (int i = 0; i < (int)this->projection.size(); i++)
if (this->projection[i].aggregate_type != ProjectionVar::None_type)
return true;
return false;
return this->group_by;
}
void QueryTree::addOrder(string &_var, bool _descending)
void QueryTree::addOrderVar(string &_var, bool _descending)
{
this->order.push_back(Order(_var, _descending));
this->order_by.push_back(Order(_var, _descending));
}
vector<QueryTree::Order>& QueryTree::getOrder()
vector<QueryTree::Order>& QueryTree::getOrderVarVector()
{
return this->order;
return this->order_by;
}
Varset QueryTree::getOrderByVarset()
{
Varset varset;
for (int i = 0; i < (int)this->order_by.size(); i++)
varset.addVar(this->order_by[i].var);
return varset;
}
void QueryTree::setOffset(int _offset)
@ -528,102 +564,135 @@ QueryTree::GroupPattern& QueryTree::getDeletePatterns()
bool QueryTree::checkWellDesigned()
{
if (!this->getGroupPattern().checkOnlyUnionOptionalFilterNoExists())
return false;
bool check_condition = true;
this->getGroupPattern().checkOptionalGroupPatternVarsAndSafeFilter(Varset(), Varset(), check_condition);
this->grouppattern.checkNoMinusAndOptionalVarAndSafeFilter(Varset(), Varset(), check_condition);
return check_condition;
}
bool QueryTree::checkAtLeastOneAggregateFunction()
{
for (int i = 0; i < (int)this->projection.size(); i++)
if (this->projection[i].aggregate_type != ProjectionVar::None_type)
return true;
return false;
}
bool QueryTree::checkSelectAggregateFunctionGroupByValid()
{
if (this->checkAtLeastOneAggregateFunction() && this->group_by.empty())
{
for (int i = 0; i < (int)this->projection.size(); i++)
if (this->projection[i].aggregate_type == ProjectionVar::None_type)
return false;
}
if (!this->group_by.empty())
{
for (int i = 0; i < (int)this->projection.size(); i++)
if (this->projection[i].aggregate_type == ProjectionVar::None_type && !this->group_by.findVar(this->projection[i].var))
return false;
}
return true;
}
void QueryTree::print()
{
for (int j = 0; j < 80; j++) printf("="); printf("\n");
if (this->getUpdateType() == Not_Update)
if (this->update_type == Not_Update)
{
if (this->getQueryForm() == Select_Query)
if (this->query_form == Select_Query)
{
printf("SELECT");
if (this->getProjectionModifier() == Modifier_Distinct)
if (this->projection_modifier == Modifier_Distinct)
printf(" DISTINCT");
printf("\n");
printf("Var: \t");
vector<ProjectionVar> &proj = this->getProjection();
for (int i = 0; i < (int)proj.size(); i++)
for (int i = 0; i < (int)this->projection.size(); i++)
{
if (proj[i].aggregate_type == QueryTree::ProjectionVar::None_type)
printf("%s\t", proj[i].var.c_str());
if (this->projection[i].aggregate_type == QueryTree::ProjectionVar::None_type)
printf("%s\t", this->projection[i].var.c_str());
else
{
printf("(");
if (proj[i].aggregate_type == QueryTree::ProjectionVar::Count_type)
if (this->projection[i].aggregate_type == QueryTree::ProjectionVar::Count_type)
printf("COUNT(");
if (proj[i].aggregate_type == QueryTree::ProjectionVar::Sum_type)
if (this->projection[i].aggregate_type == QueryTree::ProjectionVar::Sum_type)
printf("SUM(");
if (proj[i].aggregate_type == QueryTree::ProjectionVar::Min_type)
if (this->projection[i].aggregate_type == QueryTree::ProjectionVar::Min_type)
printf("MIN(");
if (proj[i].aggregate_type == QueryTree::ProjectionVar::Max_type)
if (this->projection[i].aggregate_type == QueryTree::ProjectionVar::Max_type)
printf("MAX(");
if (proj[i].aggregate_type == QueryTree::ProjectionVar::Avg_type)
if (this->projection[i].aggregate_type == QueryTree::ProjectionVar::Avg_type)
printf("AVG(");
if (proj[i].distinct)
if (this->projection[i].distinct)
printf("DISTINCT ");
printf("%s) AS %s)\t", proj[i].aggregate_var.c_str(), proj[i].var.c_str());
printf("%s) AS %s)\t", this->projection[i].aggregate_var.c_str(), this->projection[i].var.c_str());
}
}
if (this->checkProjectionAsterisk() && !this->atLeastOneAggregateFunction())
if (this->projection_asterisk && !this->checkAtLeastOneAggregateFunction())
printf("*");
printf("\n");
}
else printf("ASK\n");
printf("GroupPattern:\n");
this->getGroupPattern().print(0);
this->grouppattern.print(0);
if (this->getQueryForm() == Select_Query)
if (this->query_form == Select_Query)
{
if ((int)this->getOrder().size() > 0)
if (!this->group_by.empty())
{
printf("GROUP BY\t");
for (int i = 0; i < (int)this->group_by.vars.size(); i++)
printf("%s\t", this->group_by.vars[i].c_str());
printf("\n");
}
if (!this->order_by.empty())
{
printf("ORDER BY\t");
vector<QueryTree::Order>&order = this->getOrder();
for (int i = 0; i < (int)order.size(); i++)
for (int i = 0; i < (int)this->order_by.size(); i++)
{
if (!order[i].descending) printf("ASC(");
if (!this->order_by[i].descending) printf("ASC(");
else printf("DESC(");
printf("%s)\t", order[i].var.c_str());
printf("%s)\t", this->order_by[i].var.c_str());
}
printf("\n");
}
if (this->getOffset() != 0)
printf("OFFSET\t%d\n", this->getOffset());
if (this->getLimit() != -1)
printf("LIMIT\t%d\n", this->getLimit());
if (this->offset != 0)
printf("OFFSET\t%d\n", this->offset);
if (this->limit != -1)
printf("LIMIT\t%d\n", this->limit);
}
}
else
{
printf("UPDATE\n");
if (this->getUpdateType() == Delete_Data || this->getUpdateType() == Delete_Where ||
this->getUpdateType() == Delete_Clause || this->getUpdateType() == Modify_Clause)
if (this->update_type == Delete_Data || this->update_type == Delete_Where ||
this->update_type == Delete_Clause || this->update_type == Modify_Clause)
{
printf("Delete:\n");
this->getDeletePatterns().print(0);
this->delete_patterns.print(0);
}
if (this->getUpdateType() == Insert_Data || this->getUpdateType() == Insert_Clause || this->getUpdateType() == Modify_Clause)
if (this->update_type == Insert_Data || this->update_type == Insert_Clause || this->update_type == Modify_Clause)
{
printf("Insert:\n");
this->getInsertPatterns().print(0);
this->insert_patterns.print(0);
}
if (this->getUpdateType() == Delete_Where || this->getUpdateType() == Insert_Clause ||
this->getUpdateType() == Delete_Clause || this->getUpdateType() == Modify_Clause)
if (this->update_type == Delete_Where || this->update_type == Insert_Clause ||
this->update_type == Delete_Clause || this->update_type == Modify_Clause)
{
printf("GroupPattern:\n");
this->getGroupPattern().print(0);
this->grouppattern.print(0);
}
}
for (int j = 0; j < 80; j++) printf("="); printf("\n");
}

237
Query/QueryTree.h
View File

@ -2,7 +2,7 @@
# Filename: QueryTree.h
# Author: Jiaqi, Chen
# Mail: chenjiaqi93@163.com
# Last Modified: 2016-07-14
# Last Modified: 2017-03-13
# Description:
=============================================================================*/
@ -15,72 +15,40 @@
class QueryTree
{
public:
QueryTree():
query_form(Select_Query), update_type(Not_Update), projection_modifier(Modifier_None), projection_asterisk(false), offset(0), limit(-1){}
enum QueryForm {Select_Query, Ask_Query};
enum ProjectionModifier {Modifier_None, Modifier_Distinct, Modifier_Reduced, Modifier_Count, Modifier_Duplicates};
class GroupPattern
{
public:
class Pattern
{
public:
class Element
{
public:
/*
enum Type { Variable, Literal, IRI };
enum SubType { None, CustomLanguage, CustomType };
Type type;
SubType subType;
std::string subTypeValue;
*/
std::string value;
Element(const std::string &_value):
value(_value){}
};
Element subject, predicate, object;
Varset varset;
Pattern(const Element _subject, const Element _predicate,const Element _object):subject(_subject), predicate(_predicate), object(_object){}
};
class Pattern;
class FilterTree;
class Bind;
class SubGroupPattern;
class GroupPatternUnions;
class OptionalOrMinusGroupPattern;
class FilterTreeNode;
class FilterTreeRoot;
std::vector<Pattern> patterns;
std::vector<GroupPatternUnions> unions;
std::vector<OptionalOrMinusGroupPattern> optionals;
std::vector<FilterTreeRoot> filters;
std::vector<std::vector<GroupPattern> > filter_exists_grouppatterns;
std::vector<SubGroupPattern> sub_grouppattern;
Varset grouppattern_resultset_minimal_varset, grouppattern_resultset_maximal_varset;
Varset grouppattern_subject_object_maximal_varset, grouppattern_predicate_maximal_varset;
std::vector<int> pattern_blockid;
void addOnePattern(Pattern _pattern);
void addOneGroupUnion();
void addOneUnion();
GroupPattern& getLastUnion();
void addOneOptionalOrMinus(char _type);
GroupPattern& getLastOptionalOrMinus();
void addOneOptional(int _type);
GroupPattern& getLastOptional();
void addOneFilterTree();
FilterTreeNode& getLastFilterTree();
void addOneExistsGroupPattern();
GroupPattern& getLastExistsGroupPattern();
void addOneFilter();
FilterTree& getLastFilter();
void addOneBind();
Bind& getLastBind();
void getVarset();
bool checkOnlyUnionOptionalFilterNoExists();
std::pair<Varset, Varset> checkOptionalGroupPatternVarsAndSafeFilter(Varset occur , Varset ban, bool &check_condition);
std::pair<Varset, Varset> checkNoMinusAndOptionalVarAndSafeFilter(Varset occur_varset, Varset ban_varset, bool &check_condition);
void initPatternBlockid();
int getRootPatternBlockID(int x);
@ -89,78 +57,115 @@ class QueryTree
void print(int dep);
};
class GroupPattern::GroupPatternUnions
class GroupPattern::Pattern
{
public:
std::vector<GroupPattern> grouppattern_vec;
int lastpattern;
GroupPatternUnions(int _lastpattern):
lastpattern(_lastpattern){}
};
class GroupPattern::OptionalOrMinusGroupPattern
{
public:
GroupPattern grouppattern;
int lastpattern, lastunions;
char type;
OptionalOrMinusGroupPattern(int _lastpattern, int _lastunions, char _type):
grouppattern(GroupPattern()), lastpattern(_lastpattern), lastunions(_lastunions), type(_type){}
};
class GroupPattern::FilterTreeNode
{
public:
enum FilterOperationType
class Element
{
None_type, Or_type, And_type, Equal_type, NotEqual_type, Less_type, LessOrEqual_type, Greater_type, GreaterOrEqual_type,
Plus_type, Minus_type, Mul_type, Div_type, Not_type, UnaryPlus_type, UnaryMinus_type, Literal_type, Variable_type, IRI_type,
Function_type, ArgumentList_type,Builtin_str_type, Builtin_lang_type, Builtin_langmatches_type, Builtin_datatype_type, Builtin_bound_type,
Builtin_sameterm_type,Builtin_isiri_type, Builtin_isblank_type, Builtin_isliteral_type, Builtin_regex_type, Builtin_in_type, Builtin_exists_type
public:
//enum Type { Variable, Literal, IRI };
//enum SubType { None, CustomLanguage, CustomType };
//Type type;
//SubType subType;
//std::string subTypeValue;
std::string value;
Element(){}
Element(const std::string &_value):value(_value){}
};
FilterOperationType oper_type;
Element subject, predicate, object;
Varset varset, subject_object_varset;
int blockid;
class FilterTreeChild;
std::vector<FilterTreeChild> child;
int exists_grouppattern_id;
FilterTreeNode():
oper_type(None_type), exists_grouppattern_id(-1){}
void getVarset(Varset &varset);
void print(std::vector<GroupPattern> &exist_grouppatterns, int dep);
Pattern():blockid(-1){}
Pattern(const Element _subject, const Element _predicate, const Element _object):
subject(_subject), predicate(_predicate), object(_object), blockid(-1){}
};
class GroupPattern::FilterTreeNode::FilterTreeChild
class GroupPattern::FilterTree
{
public:
enum FilterTreeChildNodeType {None_type, Tree_type, String_type};
FilterTreeChildNodeType node_type;
class FilterTreeNode
{
public:
enum FilterOperationType
{
None_type, Or_type, And_type, Equal_type, NotEqual_type, Less_type, LessOrEqual_type, Greater_type, GreaterOrEqual_type,
Plus_type, Minus_type, Mul_type, Div_type, Not_type, UnaryPlus_type, UnaryMinus_type, Literal_type, Variable_type, IRI_type,
Function_type, ArgumentList_type, Builtin_str_type, Builtin_lang_type, Builtin_langmatches_type, Builtin_datatype_type, Builtin_bound_type,
Builtin_sameterm_type, Builtin_isiri_type, Builtin_isuri_type, Builtin_isblank_type, Builtin_isliteral_type, Builtin_isnumeric_type,
Builtin_regex_type, Builtin_in_type, Builtin_exists_type
};
FilterOperationType oper_type;
FilterTreeNode node;
std::string arg;
int pos;
bool isel;
class FilterTreeChild;
FilterTreeChild():
node_type(None_type), pos(-1), isel(true){}
};
std::vector<FilterTreeChild> child;
FilterTreeNode():oper_type(None_type){}
void getVarset(Varset &varset);
void mapVarPos2Varset(Varset &varset, Varset &entity_literal_varset);
void print(int dep);
};
class GroupPattern::FilterTreeRoot
{
public:
FilterTreeNode root;
Varset varset;
bool done;
FilterTreeRoot():done(false){}
FilterTree():done(false){}
};
class GroupPattern::FilterTree::FilterTreeNode::FilterTreeChild
{
public:
enum FilterTreeChildNodeType {None_type, Tree_type, String_type};
FilterTreeChildNodeType node_type;
FilterTreeNode node;
std::string str;
int pos;
bool isel;
FilterTreeChild():node_type(None_type), pos(-1), isel(true){}
};
class GroupPattern::Bind
{
public:
Bind(){}
Bind(const std::string &_str, const std::string &_var):str(_str), var(_var){}
std::string str, var;
Varset varset;
};
class GroupPattern::SubGroupPattern
{
public:
enum SubGroupPatternType{Pattern_type, Union_type, Optional_type, Minus_type, Filter_type, Bind_type};
SubGroupPatternType type;
Pattern pattern;
std::vector<GroupPattern> unions;
GroupPattern optional;
FilterTree filter;
Bind bind;
SubGroupPattern(SubGroupPatternType _type):type(_type){}
SubGroupPattern(const SubGroupPattern& _sgp):type(_sgp.type)
{
pattern = _sgp.pattern;
unions = _sgp.unions;
optional = _sgp.optional;
filter = _sgp.filter;
bind = _sgp.bind;
}
};
class ProjectionVar
{
public:
enum AggregateType{None_type, Count_type, Sum_type, Min_type, Max_type, Avg_type};
enum AggregateType{None_type, Count_type, Sum_type, Min_type, Max_type, Avg_type};
AggregateType aggregate_type;
std::string var, aggregate_var;
@ -174,19 +179,20 @@ class QueryTree
public:
std::string var;
bool descending;
Order(std::string &_var, bool _descending):
var(_var), descending(_descending){}
Order(std::string &_var, bool _descending):var(_var), descending(_descending){}
};
enum UpdateType {Not_Update, Insert_Data, Delete_Data, Delete_Where, Insert_Clause, Delete_Clause, Modify_Clause};
enum UpdateType {Not_Update, Insert_Data, Delete_Data, Delete_Where, Insert_Clause, Delete_Clause, Modify_Clause};
private:
QueryForm query_form;
ProjectionModifier projection_modifier;
std::vector<ProjectionVar> projection;
Varset projection_useful_varset;
bool projection_asterisk;
std::vector<Order> order;
Varset group_by;
std::vector<Order> order_by;
int offset, limit;
GroupPattern grouppattern;
@ -199,22 +205,31 @@ class QueryTree
GroupPattern insert_patterns, delete_patterns;
public:
QueryTree():
query_form(Select_Query), projection_modifier(Modifier_None), projection_asterisk(false), offset(0), limit(-1), update_type(Not_Update){}
void setQueryForm(QueryForm _queryform);
QueryForm getQueryForm();
void setProjectionModifier(ProjectionModifier _projection_modifier);
ProjectionModifier getProjectionModifier();
void addProjectionVar();
ProjectionVar& getLastProjectionVar();
std::vector<ProjectionVar>& getProjection();
std::vector<std::string> getProjectionVar();
void addProjectionUsefulVar(std::string &_var);
Varset& getProjectionUsefulVar();
Varset getProjectionVarset();
Varset getResultProjectionVarset();
void setProjectionAsterisk();
bool checkProjectionAsterisk();
bool checkSelectCompatibility();
bool atLeastOneAggregateFunction();
void addOrder(std::string &_var, bool _descending);
std::vector<Order>& getOrder();
void addGroupByVar(std::string &_var);
Varset& getGroupByVarset();
void addOrderVar(std::string &_var, bool _descending);
std::vector<Order>& getOrderVarVector();
Varset getOrderByVarset();
void setOffset(int _offset);
int getOffset();
void setLimit(int _limit);
@ -228,6 +243,8 @@ class QueryTree
GroupPattern& getDeletePatterns();
bool checkWellDesigned();
bool checkAtLeastOneAggregateFunction();
bool checkSelectAggregateFunctionGroupByValid();
void print();
};

14
Query/ResultFilter.cpp
View File

@ -23,8 +23,8 @@ void ResultFilter::changeResultHashTable(SPARQLquery &query, int value)
for (int i = 0; i < query.getBasicQueryNum(); i++)
{
BasicQuery &basicquery = query.getBasicQuery(i);
vector<int*> &basicquery_result =basicquery.getResultList();
int result_num = basicquery_result.size();
vector<unsigned*> &basicquery_result = basicquery.getResultList();
unsigned result_num = basicquery_result.size();
int var_num = basicquery.getVarNum();
for (int j = 0; j < var_num; j++)
@ -37,7 +37,7 @@ void ResultFilter::changeResultHashTable(SPARQLquery &query, int value)
refer.push_back(&this->result_filter[basicquery.getVarName(j)].second);
}
for (int j = 0; j < result_num; j++)
for (unsigned j = 0; j < result_num; j++)
for (int k = 0; k < var_num; k++)
{
(*refer[k])[this->hash(basicquery_result[j][k])] += value;
@ -45,7 +45,7 @@ void ResultFilter::changeResultHashTable(SPARQLquery &query, int value)
}
long tv_end = Util::get_cur_time();
printf("after ResultFilter::change, used %d ms.\n", tv_end - tv_begin);
printf("after ResultFilter::change, used %ld ms.\n", tv_end - tv_begin);
}
void ResultFilter::candFilterWithResultHashTable(BasicQuery &basicquery)
@ -64,9 +64,9 @@ void ResultFilter::candFilterWithResultHashTable(BasicQuery &basicquery)
printf("before candFilter, size = %d\n", idlist.size());
long tv_begin = Util::get_cur_time();
for (int k = 0; k < idlist.size(); k++)
for (unsigned k = 0; k < idlist.size(); k++)
{
int id = idlist.getID(k);
unsigned id = idlist.getID(k);
if (col[hash(id)] > 0)
{
new_idlist.addID(id);
@ -75,7 +75,7 @@ void ResultFilter::candFilterWithResultHashTable(BasicQuery &basicquery)
idlist = new_idlist;
long tv_end = Util::get_cur_time();
printf("after candFilter, size = %d, used %d ms.\n", idlist.size(), tv_end - tv_begin);
printf("after candFilter, size = %d, used %ld ms.\n", idlist.size(), tv_end - tv_begin);
}
}
}

4
Query/ResultFilter.h
View File

@ -16,8 +16,8 @@
class ResultFilter
{
private:
static const int MAX_SIZE = 1048576;
inline int hash(unsigned int x)
static const unsigned MAX_SIZE = 1048576;
inline unsigned hash(unsigned x)
{
x = (x + 0x7ed55d16) + (x << 12);
x = (x ^ 0xc761c23c) ^ (x >> 19);

26
Query/ResultSet.cpp
View File

@ -27,7 +27,7 @@ ResultSet::~ResultSet()
delete[] this->var_name;
if (!this->useStream)
{
for(int i = 0; i < this->ansNum; i++)
for(unsigned i = 0; i < this->ansNum; i++)
{
delete[] this->answer[i];
}
@ -89,9 +89,9 @@ ResultSet::setVar(const vector<string> & _var_names)
string
ResultSet::to_str()
{
int ans_num = max(this->ansNum - this->output_offset, 0);
long long ans_num = max((long long)this->ansNum - this->output_offset, 0LL);
if (this->output_limit != -1)
ans_num = min(ans_num, this->output_limit);
ans_num = min(ans_num, (long long)this->output_limit);
if(ans_num == 0)
{
return "[empty result]\n";
@ -110,8 +110,8 @@ ResultSet::to_str()
if (this->useStream)
this->resetStream();
const Bstr* bp;
for(int i = (!this->useStream ? this->output_offset : 0); i < this->ansNum; i++)
const Bstr* bp = NULL;
for(long long i = (!this->useStream ? this->output_offset : 0LL); i < this->ansNum; i++)
{
if (this->output_limit != -1 && i == this->output_offset + this->output_limit)
break;
@ -160,8 +160,8 @@ ResultSet::to_JSON()
if (this->useStream)
this->resetStream();
const Bstr* bp;
for(int i = (!this->useStream ? this->output_offset : 0); i < this->ansNum; i++)
const Bstr* bp = NULL;
for(long long i = (!this->useStream ? this->output_offset : 0LL); i < this->ansNum; i++)
{
if (this->output_limit != -1 && i == this->output_offset + this->output_limit)
break;
@ -199,14 +199,14 @@ ResultSet::to_JSON()
_buf << "\"" + this->var_name[j].substr(1) + "\": { ";
_buf << "\"type\": \"" + ans_type + "\", \"value\": \"" + Util::node2string(ans_str.c_str()) + "\" }";
}
else if (ans_str[0] == '"' && ans_str.find("\"^^<") == -1 && ans_str[ans_str.length() - 1] != '>' )
else if (ans_str[0] == '"' && ans_str.find("\"^^<") == string::npos && ans_str[ans_str.length() - 1] != '>' )
{
ans_type = "literal";
ans_str = ans_str.substr(1, ans_str.rfind('"') - 1);
_buf << "\"" + this->var_name[j].substr(1) + "\": { ";
_buf << "\"type\": \"" + ans_type + "\", \"value\": \"" + Util::node2string(ans_str.c_str()) + "\" }";
}
else if (ans_str[0] == '"' && ans_str.find("\"^^<") != -1 && ans_str[ans_str.length() - 1] == '>' )
else if (ans_str[0] == '"' && ans_str.find("\"^^<") != string::npos && ans_str[ans_str.length() - 1] == '>' )
{
ans_type = "typed-literal";
int pos = ans_str.find("\"^^<");
@ -234,9 +234,9 @@ ResultSet::output(FILE* _fp)
{
if (this->useStream)
{
int ans_num = max(this->ansNum - this->output_offset, 0);
long long ans_num = max((long long)this->ansNum - this->output_offset, 0LL);
if (this->output_limit != -1)
ans_num = min(ans_num, this->output_limit);
ans_num = min(ans_num, (long long)this->output_limit);
if(ans_num == 0)
{
fprintf(_fp, "[empty result]\n");
@ -251,7 +251,7 @@ ResultSet::output(FILE* _fp)
fprintf(_fp, "\n");
const Bstr* bp;
for(int i = 0; i < this->ansNum; i++)
for(long long i = 0; i < this->ansNum; i++)
{
if (this->output_limit != -1 && i == this->output_offset + this->output_limit)
break;
@ -275,7 +275,7 @@ ResultSet::output(FILE* _fp)
}
void
ResultSet::openStream(std::vector<int> &_keys, std::vector<bool> &_desc)
ResultSet::openStream(vector<unsigned> &_keys, vector<bool> &_desc)
{
if (this->useStream)
{

6
Query/ResultSet.h
View File

@ -20,8 +20,8 @@ private:
bool useStream;
public:
int select_var_num;
std::string* var_name;
int ansNum;
std::string* var_name;
unsigned ansNum;
std::string** answer;
int output_offset, output_limit;
@ -44,7 +44,7 @@ public:
void setVar(const std::vector<std::string> & _var_names);
//operations on private stream from caller
void openStream(std::vector<int> &_keys, std::vector<bool> &_desc);
void openStream(std::vector<unsigned> &_keys, std::vector<bool> &_desc);
void resetStream();
void writeToStream(std::string& _s);
const Bstr* getOneRecord();

170
Query/Varset.cpp
View File

@ -2,7 +2,7 @@
# Filename: Varset.cpp
# Author: Jiaqi, Chen
# Mail: chenjiaqi93@163.com
# Last Modified: 2016-03-02 20:35
# Last Modified: 2017-03-10
# Description: implement functions in Varset.h
=============================================================================*/
@ -10,100 +10,138 @@
using namespace std;
Varset::Varset(string & _var)
Varset::Varset(const string &_var)
{
addVar(_var);
this->addVar(_var);
}
Varset::Varset(vector<string> & _varset)
Varset::Varset(const vector<string> &_vars)
{
for (int i = 0; i < (int)_varset.size(); i++)
addVar(_varset[i]);
for (int i = 0; i < (int)_vars.size(); i++)
this->addVar(_vars[i]);
}
bool Varset::findVar(string& _var)
bool Varset::empty() const
{
if ((int)this->varset.size() == 0) return false;
vector<string>::iterator i = find(this->varset.begin(), this->varset.end(), _var);
return (i != this->varset.end());
return this->vars.empty();
}
void Varset::addVar(string& _var)
int Varset::getVarsetSize() const
{
if (!this->findVar(_var))
this->varset.push_back(_var);
return this->vars.size();
}
Varset Varset::operator + (Varset& x)
bool Varset::findVar(const string &_var) const
{
Varset r;
for (int i = 0; i < (int)this->varset.size(); i++)
r.addVar(this->varset[i]);
for (int i = 0; i < (int)x.varset.size(); i++)
r.addVar(x.varset[i]);
return r;
};
if (this->vars.empty())
return false;
Varset Varset::operator * (Varset& x)
{
Varset r;
for (int i = 0; i < (int)this->varset.size(); i++)
if (x.findVar(this->varset[i]))
r.addVar(this->varset[i]);
return r;
}
for (int i = 0; i < (int)this->vars.size(); i++)
if (this->vars[i] == _var) return true;
Varset Varset::operator - (Varset& x)
{
Varset r;
for (int i = 0; i < (int)this->varset.size(); i++)
if (!x.findVar(this->varset[i]))
r.addVar(this->varset[i]);
return r;
}
bool Varset::operator ==(Varset &x)
{
if ((int)this->varset.size() != (int)x.varset.size()) return false;
for (int i = 0; i < (int)this->varset.size(); i++)
if (!x.findVar(this->varset[i])) return false;
return true;
}
bool Varset::hasCommonVar(Varset &x)
{
for (int i = 0; i < (int)this->varset.size(); i++)
if (x.findVar(this->varset[i])) return true;
return false;
}
bool Varset::belongTo(Varset &x)
void Varset::addVar(const string &_var)
{
for (int i = 0; i < (int)this->varset.size(); i++)
if (!x.findVar(this->varset[i])) return false;
return true;
if (!this->findVar(_var))
this->vars.push_back(_var);
}
vector <int> Varset::mapTo(Varset& x)
Varset Varset::operator + (const Varset &_varset) const
{
vector<int> r;
for (int i = 0; i < (int)this->varset.size(); i++)
{
r.push_back(-1);
for (int j = 0; j < (int)x.varset.size(); j++)
if (this->varset[i] == x.varset[j])
r[i] = j;
}
Varset r(*this);
for (int i = 0; i < (int)_varset.vars.size(); i++)
r.addVar(_varset.vars[i]);
return r;
};
Varset& Varset::operator += (const Varset &_varset)
{
for (int i = 0; i < (int)_varset.vars.size(); i++)
this->addVar(_varset.vars[i]);
return *this;
}
Varset Varset::operator * (const Varset &_varset) const
{
Varset r;
for (int i = 0; i < (int)this->vars.size(); i++)
if (_varset.findVar(this->vars[i]))
r.addVar(this->vars[i]);
return r;
}
void Varset::print()
Varset Varset::operator - (const Varset &_varset) const
{
Varset r;
for (int i = 0; i < (int)this->vars.size(); i++)
if (!_varset.findVar(this->vars[i]))
r.addVar(this->vars[i]);
return r;
}
bool Varset::operator == (const Varset &_varset) const
{
if ((int)this->vars.size() != (int)_varset.vars.size())
return false;
for (int i = 0; i < (int)this->vars.size(); i++)
if (!_varset.findVar(this->vars[i]))
return false;
return true;
}
bool Varset::hasCommonVar(const Varset &_varset) const
{
for (int i = 0; i < (int)this->vars.size(); i++)
if (_varset.findVar(this->vars[i]))
return true;
return false;
}
bool Varset::belongTo(const Varset &_varset) const
{
for (int i = 0; i < (int)this->vars.size(); i++)
if (!_varset.findVar(this->vars[i]))
return false;
return true;
}
vector<int> Varset::mapTo(const Varset &_varset) const
{
vector<int> r;
for (int i = 0; i < (int)this->vars.size(); i++)
{
r.push_back(-1);
for (int j = 0; j < (int)_varset.vars.size(); j++)
if (this->vars[i] == _varset.vars[j])
r[i] = j;
}
return r;
}
void Varset::print() const
{
printf("Varset: ");
for (int i = 0; i < (int)this->varset.size(); i++)
for (int i = 0; i < (int)this->vars.size(); i++)
{
printf("%s ", this->varset[i].c_str());
printf("%s ", this->vars[i].c_str());
}
printf("\n");
}

33
Query/Varset.h
View File

@ -2,7 +2,7 @@
# Filename: Varset.h
# Author: Jiaqi, Chen
# Mail: chenjiaqi93@163.com
# Last Modified: 2016-03-02 20:35
# Last Modified: 2017-03-10
# Description:
=============================================================================*/
@ -14,24 +14,29 @@
class Varset
{
public:
std::vector <std::string> varset;
std::vector<std::string> vars;
Varset(){};
Varset(std::string & _var);
Varset(std::vector<std::string> & _varset);
Varset(const std::string &_var);
Varset(const std::vector<std::string> &_vars);
bool findVar(std::string& _var);
void addVar(std::string& _var);
bool empty() const;
int getVarsetSize() const;
bool findVar(const std::string &_var) const;
void addVar(const std::string &_var);
Varset operator + (Varset& x);
Varset operator * (Varset& x);
Varset operator - (Varset& x);
bool operator ==(Varset &x);
bool hasCommonVar(Varset &x);
bool belongTo(Varset &x);
Varset operator + (const Varset &_varset) const;
Varset& operator += (const Varset &_varset);
Varset operator * (const Varset &_varset) const;
Varset operator - (const Varset &_varset) const;
std::vector <int> mapTo(Varset& x);
bool operator ==(const Varset &_varset) const;
bool hasCommonVar(const Varset &_varset) const;
bool belongTo(const Varset &_varset) const;
void print();
std::vector<int> mapTo(const Varset &_varset) const;
void print() const;
};
#endif // _QUERY_VARSET_H

8
README.md
View File

@ -2,7 +2,7 @@
Gstore System(also called gStore) is a graph database engine for managing large graph-structured data, which is open-source and targets at Linux operation systems. The whole project is written in C++, with the help of some libraries such as readline, antlr, and so on. Only source tarballs are provided currently, which means you have to compile the source code if you want to use our system.
**The formal help document is in [Handbook](docs/latex/gStore_help.pdf).**
**The formal help document is in [EN](docs/latex/gStore_help.pdf) and [CH](docs/latex/gStore_help_CN.pdf).**
**The formal experiment result is in [Experiment](docs/latex/formal_experiment.pdf).**
@ -20,7 +20,7 @@ This system is really user-friendly and you can pick it up in several minutes. R
- type `git clone git@github.com:Caesar11/gStore.git` in your terminal or use git GUI to acquire it
Then you need to compile the project, just type `make` in the gStore root directory, and all executables will be ok. To run gStore, please type `bin/gbuild database_name dataset_path` to build a database named by yourself. And you can use `bin/gquery database_name` command to query a existing database. What is more, `bin/gconsole` is a wonderful tool designed for you, providing all operations you need to use gStore. Notice that all commands should be typed in the root directory of gStore, and your database name should not end with ".db".
Then you need to compile the project, just type `make` in the gStore root directory, and all executables will be ok. To run gStore, please type `bin/gbuild database_name dataset_path` to build a database named by yourself. And you can use `bin/gquery database_name` command to query a existing database. What is more, `bin/ghttp` is a wonderful tool designed for you, as a database server which can be visited via HTTP protocol. Notice that all commands should be typed in the root directory of gStore, and your database name should not end with ".db".
- - -
@ -50,7 +50,7 @@ If you want to understand the details of the gStore system, or you want to try s
We have written a series of short essays addressing recurring challenges in using gStore to realize applications, which are placed in [Recipe Book](docs/TIPS.md).
You are welcome to report any advice or errors in the github Issues part of this repository, if not requiring in-time reply. However, if you want to urgent on us to deal with your reports, please email to <chenjiaqi93@163.com> to submit your suggestions and report bugs to us by emailing to <zengli-bookug@pku.edu.cn>. A full list of our whole team is in [Mailing List](docs/MAIL.md).
You are welcome to report any advice or errors in the github Issues part of this repository, if not requiring in-time reply. However, if you want to urgent on us to deal with your reports, please email to <zengli@bookug.cc> to submit your suggestions and report bugs to us by emailing to <gStoreDB@gmail.com>. A full list of our whole team is in [Mailing List](docs/MAIL.md).
There are some restrictions when you use the current gStore project, you can see them on [Limit Description](docs/LIMIT.md).
@ -68,5 +68,5 @@ Graph database engine is a new area and we are still trying to go further. Thing
People who inspire us or contribute to this project will be listed in the [Thanks List](docs/THANK.md) chapter.
This whole document is divided into different pieces, and each them is stored in a markdown file. You can see/download the combined markdown file in [help_markdown](docs/gStore_help.md), and for html file, please go to [help_html](docs/gStore_help.html). What is more, we also provide help file in pdf format, and you can visit it in [help_pdf](docs/latex/gStore_help.pdf).
<!--This whole document is divided into different pieces, and each them is stored in a markdown file. You can see/download the combined markdown file in [help_markdown](docs/gStore_help.md), and for html file, please go to [help_html](docs/gStore_help.html). What is more, we also provide help file in pdf format, and you can visit it in [help_pdf](docs/latex/gStore_help.pdf).-->

21
Server/LICENSE Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014-2016 Ole Christian Eidheim
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

8
Server/Operation.h
View File

@ -14,8 +14,10 @@
//NOTICE:CMD_DROP is used to remove the database, and CMD_CREATE is not useful because
//we always need to import a dataset to create a gstore db
enum CommandType {
CMD_CONNECT, CMD_EXIT, CMD_TEST, CMD_LOAD, CMD_UNLOAD, CMD_CREATE, CMD_DROP,
CMD_IMPORT, CMD_QUERY, CMD_SHOW, CMD_INSERT, CMD_STOP, CMD_OTHER
CMD_CONNECT, CMD_TEST, CMD_LOAD, CMD_UNLOAD, CMD_CREATE, CMD_DROP, CMD_IMPORT,
CMD_QUERY, CMD_SHOW, CMD_INSERT, CMD_STOP, CMD_BACKUP, CMD_OTHER
//CMD_CONNECT, CMD_EXIT, CMD_TEST, CMD_LOAD, CMD_UNLOAD, CMD_CREATE, CMD_DROP,
//CMD_IMPORT, CMD_QUERY, CMD_SHOW, CMD_INSERT, CMD_STOP, CMD_OTHER
}; // extend the operation command type here.
class Operation
@ -47,4 +49,4 @@ private:
std::vector<std::string> parameters;
};
#endif /* OPERATION_H_ */
#endif /* OPERATION_H_ */

8
Server/README.md Normal file
View File

@ -0,0 +1,8 @@
A Http interface for gStore
=================
Based on Simple-Web-Server (https://travis-ci.org/eidheim/Simple-Web-Server)
To see the whole project about gStore, click at https://github.com/Caesar11/gStore

141
Server/Server.cpp
View File

@ -10,15 +10,15 @@
using namespace std;
Server::Server()
{
this->connectionPort = Socket::DEFAULT_CONNECT_PORT;
this->connectionMaxNum = Socket::MAX_CONNECTIONS;
this->databaseMaxNum = 1; // will be updated when supporting multiple databases.
this->database = NULL;
this->db_home = Util::global_config["db_home"];
this->db_suffix = Util::global_config["db_suffix"];
}
//Server::Server()
//{
//this->connectionPort = Socket::DEFAULT_CONNECT_PORT;
//this->connectionMaxNum = Socket::MAX_CONNECTIONS;
//this->databaseMaxNum = 1; // will be updated when supporting multiple databases.
//this->database = NULL;
//this->db_home = Util::global_config["db_home"];
//this->db_suffix = Util::global_config["db_suffix"];
//}
Server::Server(unsigned short _port)
{
@ -26,6 +26,10 @@ Server::Server(unsigned short _port)
this->connectionMaxNum = Socket::MAX_CONNECTIONS;
this->databaseMaxNum = 1; // will be updated when supporting multiple databases.
this->database = NULL;
this->db_home = Util::global_config["db_home"];
this->db_suffix = Util::global_config["db_suffix"];
this->next_backup = 0;
this->scheduler_pid = 0;
}
Server::~Server()
@ -151,7 +155,14 @@ Server::listen()
case CMD_QUERY:
{
string query = operation.getParameter(0);
pthread_t timer = Server::start_timer();
if (timer == 0) {
cerr << Util::getTimeString() << "Failed to start timer." << endl;
}
this->query(query, ret_msg);
if (timer != 0 && !Server::stop_timer(timer)) {
cerr << Util::getTimeString() << "Failed to stop timer." << endl;
}
break;
}
case CMD_SHOW:
@ -180,6 +191,25 @@ Server::listen()
_stop = true;
break;
}
case CMD_BACKUP: {
string para = operation.getParameter(0);
stringstream ss(para);
long time_backup;
ss >> time_backup;
if (this->next_backup == 0) {
break;
}
while (this->next_backup < time_backup) {
this->next_backup += Util::gserver_backup_interval;
}
if (this->next_backup == time_backup) {
this->backup(ret_msg);
}
else {
ret_msg = "done";
}
break;
}
default:
cerr << Util::getTimeString() << "this command is not supported by now. @Server::listen" << endl;
}
@ -191,6 +221,13 @@ Server::listen()
cout << Util::getTimeString() << "server stopped." << endl;
break;
}
if (this->next_backup > 0) {
time_t cur_time = time(NULL);
if (cur_time >= this->next_backup) {
string str;
this->backup(str);
}
}
}
}
@ -269,6 +306,10 @@ Server::parser(std::string _raw_cmd, Operation& _ret_oprt)
_ret_oprt.setCommand(CMD_STOP);
para_cnt = 0;
}
else if (cmd == "backup") {
_ret_oprt.setCommand(CMD_BACKUP);
para_cnt = 1;
}
else
{
return false;
@ -380,9 +421,36 @@ Server::loadDatabase(std::string _db_name, std::string _ac_name, std::string& _r
_ret_msg = "load database failed.";
delete this->database;
this->database = NULL;
return false;
}
return flag;
pid_t fpid = vfork();
// child, scheduler
if (fpid == 0) {
time_t cur_time = time(NULL);
long time_backup = Util::read_backup_time();
long first_backup = cur_time - (cur_time - time_backup) % Util::gserver_backup_interval
+ Util::gserver_backup_interval;
this->next_backup = first_backup;
string s_port = Util::int2string(this->connectionPort);
string s_next_backup = Util::int2string(first_backup);
execl("bin/gserver_backup_scheduler", "gserver_backup_scheduler", s_port.c_str(), s_next_backup.c_str(), NULL);
exit(0);
return true;
}
// parent
if (fpid > 0) {
this->scheduler_pid = fpid;
}
// fork failure
else if (fpid < 0) {
cerr << Util::getTimeString() << "Database will not be backed-up automatically." << endl;
}
//_ret_msg = "load database done.";
return true;
//return flag;
}
bool
@ -398,6 +466,13 @@ Server::unloadDatabase(std::string _db_name, std::string _ac_name, std::string&
this->database = NULL;
_ret_msg = "unload database done.";
this->next_backup = 0;
//string cmd = "kill " + Util::int2string(this->scheduler_pid);
//system(cmd.c_str());
kill(this->scheduler_pid, SIGTERM);
waitpid(this->scheduler_pid, NULL, 0);
this->scheduler_pid = 0;
return true;
}
@ -458,8 +533,9 @@ Server::importRDF(std::string _db_name, std::string _ac_name, std::string _rdf_p
bool
Server::query(const string _query, string& _ret_msg)
{
cout<<"Server query()"<<endl;
cout<<_query<<endl;
//cout<<"Server query()"<<endl;
//cout<<_query<<endl;
cout << Util::getTimeString() << "Server query(): " << _query << endl;
if (this->database == NULL)
{
@ -468,8 +544,8 @@ Server::query(const string _query, string& _ret_msg)
}
FILE* output = NULL;
string path = "logs/gserver_query.log";
#ifdef OUTPUT_QUERY_RESULT
string path = "logs/gserver_query.log";
output = fopen(path.c_str(), "w");
#endif
@ -537,7 +613,7 @@ Server::showDatabases(string _para, string _ac_name, string& _ret_msg)
return true;
}
bool Server::stopServer(std::string& _ret_msg) {
bool Server::stopServer(string& _ret_msg) {
if (this->database != NULL) {
delete this->database;
this->database = NULL;
@ -546,3 +622,40 @@ bool Server::stopServer(std::string& _ret_msg) {
return true;
}
bool Server::backup(string& _ret_msg) {
this->next_backup += Util::gserver_backup_interval;
if (this->database == NULL) {
_ret_msg = "No database in use.";
return false;
}
if (!this->database->backup()) {
_ret_msg = "Backup failed.";
return false;
}
_ret_msg = "done";
return true;
}
pthread_t Server::start_timer() {
pthread_t timer_thread;
if (pthread_create(&timer_thread, NULL, Server::timer, NULL) == 0) {
return timer_thread;
}
return 0;
}
bool Server::stop_timer(pthread_t _timer) {
return pthread_kill(_timer, SIGTERM) == 0;
}
void* Server::timer(void* _args) {
signal(SIGTERM, Server::timer_sigterm_handler);
sleep(Util::gserver_query_timeout);
cerr << Util::getTimeString() << "Query out of time." << endl;
abort();
}
void Server::timer_sigterm_handler(int _signal_num) {
pthread_exit(0);
}

12
Server/Server.h
View File

@ -32,8 +32,8 @@
class Server
{
public:
Server();
Server(unsigned short _port);
//Server();
Server(unsigned short _port = Socket::DEFAULT_CONNECT_PORT);
~Server();
bool createConnection();
@ -51,6 +51,7 @@ public:
//bool insertTriple(std::string _db_name, std::string _ac_name, std::string _rdf_path, std::string& _ret_msg);
bool query(const std::string _query, std::string& _ret_msg);
bool stopServer(std::string& _ret_msg);
bool backup(std::string& _ret_msg);
private:
unsigned short connectionPort;
@ -61,6 +62,13 @@ private:
std::string db_home;
std::string db_suffix;
static pthread_t start_timer();
static bool stop_timer(pthread_t _timer);
static void* timer(void* _args);
static void timer_sigterm_handler(int _signal_num);
long next_backup;
pid_t scheduler_pid;
};
#endif // _SERVER_SERVER_H

440
Server/client_http.hpp Normal file
View File

@ -0,0 +1,440 @@
#ifndef CLIENT_HTTP_HPP
#define CLIENT_HTTP_HPP
#include <boost/asio.hpp>
#include <boost/utility/string_ref.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/functional/hash.hpp>
#include <unordered_map>
#include <map>
#include <random>
#include <mutex>
#include <type_traits>
#ifndef CASE_INSENSITIVE_EQUALS_AND_HASH
#define CASE_INSENSITIVE_EQUALS_AND_HASH
//Based on http://www.boost.org/doc/libs/1_60_0/doc/html/unordered/hash_equality.html
class case_insensitive_equals {
public:
bool operator()(const std::string &key1, const std::string &key2) const {
return boost::algorithm::iequals(key1, key2);
}
};
class case_insensitive_hash {
public:
size_t operator()(const std::string &key) const {
std::size_t seed=0;
for(auto &c: key)
boost::hash_combine(seed, std::tolower(c));
return seed;
}
};
#endif
namespace SimpleWeb {
template <class socket_type>
class Client;
template <class socket_type>
class ClientBase {
public:
virtual ~ClientBase() {}
class Response {
friend class ClientBase<socket_type>;
friend class Client<socket_type>;
public:
std::string http_version, status_code;
std::istream content;
std::unordered_multimap<std::string, std::string, case_insensitive_hash, case_insensitive_equals> header;
private:
boost::asio::streambuf content_buffer;
Response(): content(&content_buffer) {}
};
class Config {
friend class ClientBase<socket_type>;
private:
Config() {}
public:
/// Set timeout on requests in seconds. Default value: 0 (no timeout).
size_t timeout=0;
/// Set connect timeout in seconds. Default value: 0 (Config::timeout is then used instead).
size_t timeout_connect=0;
/// Set proxy server (server:port)
std::string proxy_server;
};
/// Set before calling request
Config config;
std::shared_ptr<Response> request(const std::string& request_type, const std::string& path="/", boost::string_ref content="",
const std::map<std::string, std::string>& header=std::map<std::string, std::string>()) {
auto corrected_path=path;
if(corrected_path=="")
corrected_path="/";
if(!config.proxy_server.empty() && std::is_same<socket_type, boost::asio::ip::tcp::socket>::value)
corrected_path="http://"+host+':'+std::to_string(port)+corrected_path;
boost::asio::streambuf write_buffer;
std::ostream write_stream(&write_buffer);
write_stream << request_type << " " << corrected_path << " HTTP/1.1\r\n";
write_stream << "Host: " << host << "\r\n";
for(auto& h: header) {
write_stream << h.first << ": " << h.second << "\r\n";
}
if(content.size()>0)
write_stream << "Content-Length: " << content.size() << "\r\n";
write_stream << "\r\n";
connect();
auto timer=get_timeout_timer();
boost::asio::async_write(*socket, write_buffer,
[this, &content, timer](const boost::system::error_code &ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(!ec) {
if(!content.empty()) {
auto timer=get_timeout_timer();
boost::asio::async_write(*socket, boost::asio::buffer(content.data(), content.size()),
[this, timer](const boost::system::error_code &ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(ec) {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
io_service.reset();
io_service.run();
return request_read();
}
std::shared_ptr<Response> request(const std::string& request_type, const std::string& path, std::iostream& content,
const std::map<std::string, std::string>& header=std::map<std::string, std::string>()) {
auto corrected_path=path;
if(corrected_path=="")
corrected_path="/";
if(!config.proxy_server.empty() && std::is_same<socket_type, boost::asio::ip::tcp::socket>::value)
corrected_path="http://"+host+':'+std::to_string(port)+corrected_path;
content.seekp(0, std::ios::end);
auto content_length=content.tellp();
content.seekp(0, std::ios::beg);
boost::asio::streambuf write_buffer;
std::ostream write_stream(&write_buffer);
write_stream << request_type << " " << corrected_path << " HTTP/1.1\r\n";
write_stream << "Host: " << host << "\r\n";
for(auto& h: header) {
write_stream << h.first << ": " << h.second << "\r\n";
}
if(content_length>0)
write_stream << "Content-Length: " << content_length << "\r\n";
write_stream << "\r\n";
if(content_length>0)
write_stream << content.rdbuf();
connect();
auto timer=get_timeout_timer();
boost::asio::async_write(*socket, write_buffer,
[this, timer](const boost::system::error_code &ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(ec) {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
io_service.reset();
io_service.run();
return request_read();
}
void close() {
std::lock_guard<std::mutex> lock(socket_mutex);
if(socket) {
boost::system::error_code ec;
socket->lowest_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);
socket->lowest_layer().close();
}
}
protected:
boost::asio::io_service io_service;
boost::asio::ip::tcp::resolver resolver;
std::unique_ptr<socket_type> socket;
std::mutex socket_mutex;
std::string host;
unsigned short port;
ClientBase(const std::string& host_port, unsigned short default_port) : resolver(io_service) {
auto parsed_host_port=parse_host_port(host_port, default_port);
host=parsed_host_port.first;
port=parsed_host_port.second;
}
std::pair<std::string, unsigned short> parse_host_port(const std::string &host_port, unsigned short default_port) {
std::pair<std::string, unsigned short> parsed_host_port;
size_t host_end=host_port.find(':');
if(host_end==std::string::npos) {
parsed_host_port.first=host_port;
parsed_host_port.second=default_port;
}
else {
parsed_host_port.first=host_port.substr(0, host_end);
parsed_host_port.second=static_cast<unsigned short>(stoul(host_port.substr(host_end+1)));
}
return parsed_host_port;
}
virtual void connect()=0;
std::shared_ptr<boost::asio::deadline_timer> get_timeout_timer(size_t timeout=0) {
if(timeout==0)
timeout=config.timeout;
if(timeout==0)
return nullptr;
auto timer=std::make_shared<boost::asio::deadline_timer>(io_service);
timer->expires_from_now(boost::posix_time::seconds(timeout));
timer->async_wait([this](const boost::system::error_code& ec) {
if(!ec) {
close();
}
});
return timer;
}
void parse_response_header(const std::shared_ptr<Response> &response) const {
std::string line;
getline(response->content, line);
size_t version_end=line.find(' ');
if(version_end!=std::string::npos) {
if(5<line.size())
response->http_version=line.substr(5, version_end-5);
if((version_end+1)<line.size())
response->status_code=line.substr(version_end+1, line.size()-(version_end+1)-1);
getline(response->content, line);
size_t param_end;
while((param_end=line.find(':'))!=std::string::npos) {
size_t value_start=param_end+1;
if((value_start)<line.size()) {
if(line[value_start]==' ')
value_start++;
if(value_start<line.size())
response->header.insert(std::make_pair(line.substr(0, param_end), line.substr(value_start, line.size()-value_start-1)));
}
getline(response->content, line);
}
}
}
std::shared_ptr<Response> request_read() {
std::shared_ptr<Response> response(new Response());
boost::asio::streambuf chunked_streambuf;
auto timer=get_timeout_timer();
boost::asio::async_read_until(*socket, response->content_buffer, "\r\n\r\n",
[this, &response, &chunked_streambuf, timer](const boost::system::error_code& ec, size_t bytes_transferred) {
if(timer)
timer->cancel();
if(!ec) {
size_t num_additional_bytes=response->content_buffer.size()-bytes_transferred;
parse_response_header(response);
auto header_it=response->header.find("Content-Length");
if(header_it!=response->header.end()) {
auto content_length=stoull(header_it->second);
if(content_length>num_additional_bytes) {
auto timer=get_timeout_timer();
boost::asio::async_read(*socket, response->content_buffer,
boost::asio::transfer_exactly(content_length-num_additional_bytes),
[this, timer](const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(ec) {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
}
else if((header_it=response->header.find("Transfer-Encoding"))!=response->header.end() && header_it->second=="chunked") {
request_read_chunked(response, chunked_streambuf);
}
else if(response->http_version<"1.1" || ((header_it=response->header.find("Connection"))!=response->header.end() && header_it->second=="close")) {
auto timer=get_timeout_timer();
boost::asio::async_read(*socket, response->content_buffer,
[this, timer](const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(ec) {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
if(ec!=boost::asio::error::eof)
throw boost::system::system_error(ec);
}
});
}
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
io_service.reset();
io_service.run();
return response;
}
void request_read_chunked(const std::shared_ptr<Response> &response, boost::asio::streambuf &streambuf) {
auto timer=get_timeout_timer();
boost::asio::async_read_until(*socket, response->content_buffer, "\r\n",
[this, &response, &streambuf, timer](const boost::system::error_code& ec, size_t bytes_transferred) {
if(timer)
timer->cancel();
if(!ec) {
std::string line;
getline(response->content, line);
bytes_transferred-=line.size()+1;
line.pop_back();
std::streamsize length=stol(line, 0, 16);
auto num_additional_bytes=static_cast<std::streamsize>(response->content_buffer.size()-bytes_transferred);
auto post_process=[this, &response, &streambuf, length] {
std::ostream stream(&streambuf);
if(length>0) {
std::vector<char> buffer(static_cast<size_t>(length));
response->content.read(&buffer[0], length);
stream.write(&buffer[0], length);
}
//Remove "\r\n"
response->content.get();
response->content.get();
if(length>0)
request_read_chunked(response, streambuf);
else {
std::ostream response_stream(&response->content_buffer);
response_stream << stream.rdbuf();
}
};
if((2+length)>num_additional_bytes) {
auto timer=get_timeout_timer();
boost::asio::async_read(*socket, response->content_buffer,
boost::asio::transfer_exactly(2+length-num_additional_bytes),
[this, post_process, timer](const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(!ec) {
post_process();
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
else
post_process();
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
};
template<class socket_type>
class Client : public ClientBase<socket_type> {};
typedef boost::asio::ip::tcp::socket HTTP;
template<>
class Client<HTTP> : public ClientBase<HTTP> {
public:
Client(const std::string& server_port_path) : ClientBase<HTTP>::ClientBase(server_port_path, 80) {}
protected:
void connect() {
if(!socket || !socket->is_open()) {
std::unique_ptr<boost::asio::ip::tcp::resolver::query> query;
if(config.proxy_server.empty())
query=std::unique_ptr<boost::asio::ip::tcp::resolver::query>(new boost::asio::ip::tcp::resolver::query(host, std::to_string(port)));
else {
auto proxy_host_port=parse_host_port(config.proxy_server, 80);
query=std::unique_ptr<boost::asio::ip::tcp::resolver::query>(new boost::asio::ip::tcp::resolver::query(proxy_host_port.first, std::to_string(proxy_host_port.second)));
}
resolver.async_resolve(*query, [this](const boost::system::error_code &ec,
boost::asio::ip::tcp::resolver::iterator it){
if(!ec) {
{
std::lock_guard<std::mutex> lock(socket_mutex);
socket=std::unique_ptr<HTTP>(new HTTP(io_service));
}
auto timer=get_timeout_timer(config.timeout_connect);
boost::asio::async_connect(*socket, it, [this, timer]
(const boost::system::error_code &ec, boost::asio::ip::tcp::resolver::iterator /*it*/){
if(timer)
timer->cancel();
if(!ec) {
boost::asio::ip::tcp::no_delay option(true);
this->socket->set_option(option);
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
io_service.reset();
io_service.run();
}
}
};
}
#endif /* CLIENT_HTTP_HPP */

463
Server/server_http.hpp Normal file
View File

@ -0,0 +1,463 @@
#ifndef SERVER_HTTP_HPP
#define SERVER_HTTP_HPP
#include <boost/asio.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/functional/hash.hpp>
#include <map>
#include <unordered_map>
#include <thread>
#include <functional>
#include <iostream>
#include <sstream>
#include <memory>
#ifndef CASE_INSENSITIVE_EQUALS_AND_HASH
#define CASE_INSENSITIVE_EQUALS_AND_HASH
//Based on http://www.boost.org/doc/libs/1_60_0/doc/html/unordered/hash_equality.html
class case_insensitive_equals {
public:
bool operator()(const std::string &key1, const std::string &key2) const {
return boost::algorithm::iequals(key1, key2);
}
};
class case_insensitive_hash {
public:
size_t operator()(const std::string &key) const {
std::size_t seed=0;
for(auto &c: key)
boost::hash_combine(seed, std::tolower(c));
return seed;
}
};
#endif
// Late 2017 TODO: remove the following checks and always use std::regex
#ifdef USE_BOOST_REGEX
#include <boost/regex.hpp>
#define REGEX_NS boost
#else
#include <regex>
#define REGEX_NS std
#endif
// TODO when switching to c++14, use [[deprecated]] instead
#ifndef DEPRECATED
#ifdef __GNUC__
#define DEPRECATED __attribute__((deprecated))
#elif defined(_MSC_VER)
#define DEPRECATED __declspec(deprecated)
#else
#define DEPRECATED
#endif
#endif
namespace SimpleWeb {
template <class socket_type>
class Server;
template <class socket_type>
class ServerBase {
public:
virtual ~ServerBase() {}
class Response : public std::ostream {
friend class ServerBase<socket_type>;
boost::asio::streambuf streambuf;
std::shared_ptr<socket_type> socket;
Response(const std::shared_ptr<socket_type> &socket): std::ostream(&streambuf), socket(socket) {}
public:
size_t size() {
return streambuf.size();
}
/// If true, force server to close the connection after the response have been sent.
///
/// This is useful when implementing a HTTP/1.0-server sending content
/// without specifying the content length.
bool close_connection_after_response = false;
};
class Content : public std::istream {
friend class ServerBase<socket_type>;
public:
size_t size() {
return streambuf.size();
}
std::string string() {
std::stringstream ss;
ss << rdbuf();
return ss.str();
}
private:
boost::asio::streambuf &streambuf;
Content(boost::asio::streambuf &streambuf): std::istream(&streambuf), streambuf(streambuf) {}
};
class Request {
friend class ServerBase<socket_type>;
friend class Server<socket_type>;
public:
std::string method, path, http_version;
Content content;
std::unordered_multimap<std::string, std::string, case_insensitive_hash, case_insensitive_equals> header;
REGEX_NS::smatch path_match;
std::string remote_endpoint_address;
unsigned short remote_endpoint_port;
private:
Request(const socket_type &socket): content(streambuf) {
try {
remote_endpoint_address=socket.lowest_layer().remote_endpoint().address().to_string();
remote_endpoint_port=socket.lowest_layer().remote_endpoint().port();
}
catch(...) {}
}
boost::asio::streambuf streambuf;
};
class Config {
friend class ServerBase<socket_type>;
Config(unsigned short port): port(port) {}
public:
/// Port number to use. Defaults to 80 for HTTP and 443 for HTTPS.
unsigned short port;
/// Number of threads that the server will use when start() is called. Defaults to 1 thread.
size_t thread_pool_size=1;
/// Timeout on request handling. Defaults to 5 seconds.
size_t timeout_request=5;
/// Timeout on content handling. Defaults to 300 seconds.
size_t timeout_content=300;
/// IPv4 address in dotted decimal form or IPv6 address in hexadecimal notation.
/// If empty, the address will be any address.
std::string address;
/// Set to false to avoid binding the socket to an address that is already in use. Defaults to true.
bool reuse_address=true;
};
///Set before calling start().
Config config;
private:
class regex_orderable : public REGEX_NS::regex {
std::string str;
public:
regex_orderable(const char *regex_cstr) : REGEX_NS::regex(regex_cstr), str(regex_cstr) {}
regex_orderable(const std::string &regex_str) : REGEX_NS::regex(regex_str), str(regex_str) {}
bool operator<(const regex_orderable &rhs) const {
return str<rhs.str;
}
};
public:
/// Warning: do not add or remove resources after start() is called
std::map<regex_orderable, std::map<std::string,
std::function<void(std::shared_ptr<typename ServerBase<socket_type>::Response>, std::shared_ptr<typename ServerBase<socket_type>::Request>)> > > resource;
std::map<std::string,
std::function<void(std::shared_ptr<typename ServerBase<socket_type>::Response>, std::shared_ptr<typename ServerBase<socket_type>::Request>)> > default_resource;
std::function<void(std::shared_ptr<typename ServerBase<socket_type>::Request>, const boost::system::error_code&)> on_error;
std::function<void(std::shared_ptr<socket_type> socket, std::shared_ptr<typename ServerBase<socket_type>::Request>)> on_upgrade;
virtual void start() {
if(!io_service)
io_service=std::make_shared<boost::asio::io_service>();
if(io_service->stopped())
io_service->reset();
boost::asio::ip::tcp::endpoint endpoint;
if(config.address.size()>0)
endpoint=boost::asio::ip::tcp::endpoint(boost::asio::ip::address::from_string(config.address), config.port);
else
endpoint=boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), config.port);
if(!acceptor)
acceptor=std::unique_ptr<boost::asio::ip::tcp::acceptor>(new boost::asio::ip::tcp::acceptor(*io_service));
acceptor->open(endpoint.protocol());
acceptor->set_option(boost::asio::socket_base::reuse_address(config.reuse_address));
acceptor->bind(endpoint);
acceptor->listen();
accept();
//If thread_pool_size>1, start m_io_service.run() in (thread_pool_size-1) threads for thread-pooling
threads.clear();
for(size_t c=1;c<config.thread_pool_size;c++) {
threads.emplace_back([this]() {
io_service->run();
});
}
//Main thread
if(config.thread_pool_size>0)
io_service->run();
//Wait for the rest of the threads, if any, to finish as well
for(auto& t: threads) {
t.join();
}
}
void stop() {
acceptor->close();
if(config.thread_pool_size>0)
io_service->stop();
}
///Use this function if you need to recursively send parts of a longer message
void send(const std::shared_ptr<Response> &response, const std::function<void(const boost::system::error_code&)>& callback=nullptr) const {
boost::asio::async_write(*response->socket, response->streambuf, [this, response, callback](const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(callback)
callback(ec);
});
}
/// If you have your own boost::asio::io_service, store its pointer here before running start().
/// You might also want to set config.thread_pool_size to 0.
std::shared_ptr<boost::asio::io_service> io_service;
protected:
std::unique_ptr<boost::asio::ip::tcp::acceptor> acceptor;
std::vector<std::thread> threads;
ServerBase(unsigned short port) : config(port) {}
virtual void accept()=0;
std::shared_ptr<boost::asio::deadline_timer> get_timeout_timer(const std::shared_ptr<socket_type> &socket, long seconds) {
if(seconds==0)
return nullptr;
auto timer=std::make_shared<boost::asio::deadline_timer>(*io_service);
timer->expires_from_now(boost::posix_time::seconds(seconds));
timer->async_wait([socket](const boost::system::error_code& ec){
if(!ec) {
boost::system::error_code ec;
socket->lowest_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);
socket->lowest_layer().close();
}
});
return timer;
}
void read_request_and_content(const std::shared_ptr<socket_type> &socket) {
//Create new streambuf (Request::streambuf) for async_read_until()
//shared_ptr is used to pass temporary objects to the asynchronous functions
std::shared_ptr<Request> request(new Request(*socket));
//Set timeout on the following boost::asio::async-read or write function
auto timer=this->get_timeout_timer(socket, config.timeout_request);
boost::asio::async_read_until(*socket, request->streambuf, "\r\n\r\n",
[this, socket, request, timer](const boost::system::error_code& ec, size_t bytes_transferred) {
if(timer)
timer->cancel();
if(!ec) {
//request->streambuf.size() is not necessarily the same as bytes_transferred, from Boost-docs:
//"After a successful async_read_until operation, the streambuf may contain additional data beyond the delimiter"
//The chosen solution is to extract lines from the stream directly when parsing the header. What is left of the
//streambuf (maybe some bytes of the content) is appended to in the async_read-function below (for retrieving content).
size_t num_additional_bytes=request->streambuf.size()-bytes_transferred;
if(!this->parse_request(request))
return;
//If content, read that as well
auto it=request->header.find("Content-Length");
if(it!=request->header.end()) {
unsigned long long content_length;
try {
content_length=stoull(it->second);
}
catch(const std::exception &e) {
if(on_error)
on_error(request, boost::system::error_code(boost::system::errc::protocol_error, boost::system::generic_category()));
return;
}
if(content_length>num_additional_bytes) {
//Set timeout on the following boost::asio::async-read or write function
auto timer=this->get_timeout_timer(socket, config.timeout_content);
boost::asio::async_read(*socket, request->streambuf,
boost::asio::transfer_exactly(content_length-num_additional_bytes),
[this, socket, request, timer]
(const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(!ec)
this->find_resource(socket, request);
else if(on_error)
on_error(request, ec);
});
}
else
this->find_resource(socket, request);
}
else
this->find_resource(socket, request);
}
else if(on_error)
on_error(request, ec);
});
}
bool parse_request(const std::shared_ptr<Request> &request) const {
std::string line;
getline(request->content, line);
size_t method_end;
if((method_end=line.find(' '))!=std::string::npos) {
size_t path_end;
if((path_end=line.find(' ', method_end+1))!=std::string::npos) {
request->method=line.substr(0, method_end);
request->path=line.substr(method_end+1, path_end-method_end-1);
size_t protocol_end;
if((protocol_end=line.find('/', path_end+1))!=std::string::npos) {
if(line.compare(path_end+1, protocol_end-path_end-1, "HTTP")!=0)
return false;
request->http_version=line.substr(protocol_end+1, line.size()-protocol_end-2);
}
else
return false;
getline(request->content, line);
size_t param_end;
while((param_end=line.find(':'))!=std::string::npos) {
size_t value_start=param_end+1;
if((value_start)<line.size()) {
if(line[value_start]==' ')
value_start++;
if(value_start<line.size())
request->header.emplace(line.substr(0, param_end), line.substr(value_start, line.size()-value_start-1));
}
getline(request->content, line);
}
}
else
return false;
}
else
return false;
return true;
}
void find_resource(const std::shared_ptr<socket_type> &socket, const std::shared_ptr<Request> &request) {
//Upgrade connection
if(on_upgrade) {
auto it=request->header.find("Upgrade");
if(it!=request->header.end()) {
on_upgrade(socket, request);
return;
}
}
//Find path- and method-match, and call write_response
for(auto &regex_method: resource) {
auto it=regex_method.second.find(request->method);
if(it!=regex_method.second.end()) {
REGEX_NS::smatch sm_res;
if(REGEX_NS::regex_match(request->path, sm_res, regex_method.first)) {
request->path_match=std::move(sm_res);
write_response(socket, request, it->second);
return;
}
}
}
auto it=default_resource.find(request->method);
if(it!=default_resource.end()) {
write_response(socket, request, it->second);
}
}
void write_response(const std::shared_ptr<socket_type> &socket, const std::shared_ptr<Request> &request,
std::function<void(std::shared_ptr<typename ServerBase<socket_type>::Response>,
std::shared_ptr<typename ServerBase<socket_type>::Request>)>& resource_function) {
//Set timeout on the following boost::asio::async-read or write function
auto timer=this->get_timeout_timer(socket, config.timeout_content);
auto response=std::shared_ptr<Response>(new Response(socket), [this, request, timer](Response *response_ptr) {
auto response=std::shared_ptr<Response>(response_ptr);
this->send(response, [this, response, request, timer](const boost::system::error_code& ec) {
if(timer)
timer->cancel();
if(!ec) {
if (response->close_connection_after_response)
return;
auto range=request->header.equal_range("Connection");
for(auto it=range.first;it!=range.second;it++) {
if(boost::iequals(it->second, "close"))
return;
}
if(request->http_version >= "1.1")
this->read_request_and_content(response->socket);
}
else if(on_error)
on_error(request, ec);
});
});
try {
resource_function(response, request);
}
catch(const std::exception &e) {
if(on_error)
on_error(request, boost::system::error_code(boost::system::errc::operation_canceled, boost::system::generic_category()));
return;
}
}
};
template<class socket_type>
class Server : public ServerBase<socket_type> {};
typedef boost::asio::ip::tcp::socket HTTP;
template<>
class Server<HTTP> : public ServerBase<HTTP> {
public:
DEPRECATED Server(unsigned short port, size_t thread_pool_size=1, long timeout_request=5, long timeout_content=300) :
Server() {
config.port=port;
config.thread_pool_size=thread_pool_size;
config.timeout_request=timeout_request;
config.timeout_content=timeout_content;
}
Server() : ServerBase<HTTP>::ServerBase(9002) {}
//Server() : ServerBase<HTTP>::ServerBase(80) {}
protected:
void accept() {
//Create new socket for this connection
//Shared_ptr is used to pass temporary objects to the asynchronous functions
auto socket=std::make_shared<HTTP>(*io_service);
acceptor->async_accept(*socket, [this, socket](const boost::system::error_code& ec){
//Immediately start accepting a new connection (if io_service hasn't been stopped)
if (ec != boost::asio::error::operation_aborted)
accept();
if(!ec) {
boost::asio::ip::tcp::no_delay option(true);
socket->set_option(option);
this->read_request_and_content(socket);
}
else if(on_error)
on_error(std::shared_ptr<Request>(new Request(*socket)), ec);
});
}
};
}
#endif /* SERVER_HTTP_HPP */

37
Server/web/PHPAPIExample.php Executable file
View File

@ -0,0 +1,37 @@
<?php
printf("this is");
$result = $_POST["queryAns1"];
$format = $_POST["format"];
switch ($format) {
case 1:
$array = explode("<", $result);
$html = '<html><table class="sparql" border="1"><tr><th>' . $array[0] . "</th></tr>";
for ($i = 1; $i < count($array); $i++) {
$href = str_replace(">", "", $array[$i]);
$html.= '<tr><td><a href="' . $href . '">' . $href . '</a></td></tr>';
}
$html.= '</table></html>';
echo $html;
exit;
case 2:
$filename = 'sparql.txt';
header("Content-Type: application/octet-stream");
header('Content-Disposition: attachment; filename="' . $filename . '"');
echo $result;
exit;
case 3:
$filename = 'sparql.csv';
header("Content-Type: application/octet-stream");
header('Content-Disposition: attachment; filename="' . $filename . '"');
$array = explode("<", $result);
echo $array[0];
for ($i = 1; $i < count($array); $i++) {
$href = str_replace(">", "", $array[$i]);
echo $href;
}
exit;
}
?>

92
Server/web/admin.html Normal file
View File

@ -0,0 +1,92 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Bootstrap -->
<link rel="stylesheet" href="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
<!--[if lt IE 9]>
<script src="https://cdn.bootcss.com/html5shiv/3.7.3/html5shiv.min.js"></script>
<script src="https://cdn.bootcss.com/respond.js/1.4.2/respond.min.js"></script>
<![endif]-->
</head>
<body>
<div class="container">
<div style="margin: 50px 150px">
<!-- Nav tabs -->
<ul class="nav nav-tabs" role="tablist">
<li role="presentation" class="active"><a href="#build" aria-controls="build" role="tab" data-toggle="tab">build</a></li>
<li role="presentation"><a href="#load" aria-controls="load" role="tab" data-toggle="tab">load</a></li>
<li role="presentation"><a href="#query" aria-controls="query" role="tab" data-toggle="tab">query</a></li>
<li role="presentation"><a href="#unload" aria-controls="unload" role="tab" data-toggle="tab">unload</a></li>
</ul>
<!-- Tab panes -->
<div class="tab-content" style="margin: 20px auto">
<div role="tabpanel" class="tab-pane active" id="build">
<form>
<div class="form-group">
<label for="_db_name">Database name</label>
<input type="text" class="form-control" id="_db_name" name="_db_name" placeholder="db_name">
</div>
<div class="form-group">
<label for="_ds_path">Dataset path</label>
<input type="text" class="form-control" id="_ds_path" name="_ds_path" placeholder="ds_path">
</div>
<button id="build_button" type="button" class="btn btn-default" onclick="build(_db_name.value, _ds_path.value)">build</button>
</form>
</div>
<div role="tabpanel" class="tab-pane" id="load">
<form>
<div class="form-group">
<label for="_db_name_1">Database name</label>
<input type="text" class="form-control" id="_db_name_1" name="_db_name_1" placeholder="db_name">
</div>
<button id="load_button" type="button" class="btn btn-default" onclick="load(_db_name_1.value)">load</button>
</form>
</div>
<div role="tabpanel" class="tab-pane" id="query">
<form>
<div class="form-group">
<label for="query_file">Query file Path</label>
<input type="text" class="form-control" id="query_file" name="query_file" placeholder="query_file_path">
</div>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">query</button>
</form>
<div class="alert alert-warning alert-dismissible fade in" style="margin: 20px auto; height: 400px; overflow: auto" id="queryAns">
</div>
</div>
<div role="tabpanel" class="tab-pane" id="unload">
<form>
<div class="form-group">
<button id="unload_button" type="button" class="btn btn-default" onclick="unload()">unload</button>
</div>
</form>
</div>
</div>
</div>
</div>
</div>
<!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<script src="admin.js"></script>
</body>
</html>

34
Server/web/admin.js Normal file
View File

@ -0,0 +1,34 @@
function build(db, ds){
$.get("build/" + db + "/" + ds, function(data, status){
if(status=="success"){
alert(data);
}
});
}
1
function load(db) {
$.get("load/" + db, function(data, status){
if(status=="success"){
alert(data);
}
});
}
function query(dp) {
$.get("query/" + dp, function(data, status){
if(status=="success"){
$("#queryAns").empty();
var res = $("<p></p>").text(data);
$("#queryAns").append(res);
$("#queryAns").scrollTop($("#queryAns").height());
}
});
}
function unload(db) {
$.get("unload", function(data, status){
if(status=="success"){
alert(data);
}
});
}

82
Server/web/dbpedia/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>DBpedia Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>DBpedia Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>DBpedia Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="dbpedia" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<option value="q4">q4</option>
<option value="q5">q5</option>
<option value="q6">q6</option>
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

232
Server/web/dbpedia/index.js Normal file
View File

@ -0,0 +1,232 @@
function query(dp) {
var encodeVal = escape(dp);
//alert(encodeVal);
var format = document.getElementById("element_5").value;
if(format == "1")
format = "html";
else if(format == "2")
format = "txt";
else if(format == "3")
format = "csv";
else if(format == "4")
format = "json";
var argu = "?operation=query&format=" + format + "&sparql=" + dp;
var encodeArgu = escape(argu);
if(format != "html")
{
/*
$.get(encodeArgu, function(data, status){
if(status == "success")
{
alert("success");
}
});
*/
var element = document.getElementById("hideLink");
element.setAttribute("href", encodeArgu);
/*
if(format == "txt")
element.setAttribute("download", "sparql.txt");
else if(format == "csv")
element.setAttribute("download", "sparql.csv");
else if(format == "json")
element.setAttribute("download", "sparql.json");
*/
if(/msie/i.test(navigator.userAgent))
{
element.fireEvent("onclick");
}
else
{
var e = document.createEvent("MouseEvents");
e.initEvent("click", true, true);
element.dispatchEvent(e);
}
}
else
{
$.get(encodeArgu, function(data, status){
if(status=="success"){
//toTxt();
//alert(data);
var parts = data.split("+");
var fileName = parts[2];
var lines = Number(parts[1]);
//alert(lines);
if(lines > 100)
lines = 100;
//alert(lines);
var items = parts[3].split("\n");
//alert(items[0]);
var valNum = items[0].split("?");
var rows = valNum.length - 1;
//alert(rows);
var page = '<html><div align="left"><a href="javascript:void(0);" id="back" style="font-size:16px;color:blue">Click to Return</a>';
page = page + '<a id="download" style="font-size:16px;margin-left:20px">Click to Download</a>';
page = page + '<a href="/" id="trick" style="display: none">Click to back</a>';
page = page + '<p>Total answers: ';
page = page + parts[1];
page = page + '</p>';
if(parts[0] == "1")
{
page = page + '<p>Number of answers is too large, show only 100 of them, click to download the full result!</p>';
}
page = page + '</div><table border="1" style = "font-size:medium">';
page = page + "<tr>";
for(var ii = 1; ii <= rows; ii++)
{
page = page + "<th>" + "?" + valNum[ii] + "</th>";
}
page = page + "</tr>";
var i = 1;
var j = 0;
for (i = 1; i <= lines; i++)
{
page = page + "<tr>";
//alert(items[i]);
var tmpItem = items[i].replace(/"([^"]*)"/g, "<$1>");
//alert(tmpItem);
var item = tmpItem.split(">");
//alert(item.length);
for(j = 0; j < rows; j++)
{
//alert(item[j]);
page = page + '<td>' + item[j].replace("<", "") + '</td>';
}
page = page + "</tr>";
}
page = page + '</table></div></html>';
$("#main_body").empty();
$("#main_body").append(page);
var tmp1 = "?operation=download&filepath=" + fileName;
var request1 = escape(tmp1);
var element1 = document.getElementById("download");
element1.setAttribute("href", request1);
element1.setAttribute("download", "sparql.txt");
var tmp2 = "?operation=delete&filepath=" + fileName;
var request2 = escape(tmp2);
var element2 = document.getElementById("back");
//!Notice: element2 is a "<a>" tag, and it has two actions, href and onclick, be careful with the executing order of these two actions.
//in this case, we use a call-back function to prevent strange things. we return to the origin web page after the request to delete file returns successfully.
element2.onclick = function(){
$.get(request2, function(data, status){
//alert("delete return");
var element3 = document.getElementById("trick");
if(/msie/i.test(navigator.userAgent))
{
element3.fireEvent("onclick");
}
else
{
var e1 = document.createEvent("MouseEvents");
e1.initEvent("click", true, true);
element3.dispatchEvent(e1);
}
});
}
}
});
}
}
function handleQueryExample()
{
var example = document.getElementById("example").value;
if (example === "q1")
{
document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 WHERE \n" +
"{ \n" +
"\t?v0 <http://dbpedia.org/property/dateOfBirth> ?v2 .\n" +
"\t?v1 <http://dbpedia.org/property/genre> ?v2 .\n" +
"} \n";
}
if (example === "q2")
{
document.getElementById("element_3").value = "SELECT ?v0 WHERE \n" +
"{ \n" +
"\t?v0 <http://dbpedia.org/ontology/associatedBand> <http://dbpedia.org/resource/LCD_Soundsystem> .\n" +
"} \n";
}
if (example === "q3")
{
document.getElementById("element_3").value = "SELECT ?v2 WHERE \n" +
"{ \n" +
"\t<http://dbpedia.org/resource/!!Destroy-Oh-Boy!!> <http://dbpedia.org/property/title> ?v2 .\n" +
"} \n";
}
if (example === "q4")
{
document.getElementById("element_3").value = "SELECT ?v0 ?v2 WHERE \n" +
"{ \n" +
"\t ?v0 <http://dbpedia.org/ontology/activeYearsStartYear> ?v2 .\n" +
"} \n";
}
if (example === "q5")
{
document.getElementById("element_3").value = "SELECT ?v0 ?V1 ?V2 ?v3 WHERE \n" +
"{ \n" +
"\t?v0 <http://dbpedia.org/property/familycolor> ?v1 .\n" +
"\t?v0 <http://dbpedia.org/property/glotto> ?v2 .\n" +
"\t?v0 <http://dbpedia.org/property/lc> ?v3 .\n" +
"} \n";
}
if (example === "q6")
{
document.getElementById("element_3").value = "SELECT DISTINCT ?v0 ?v1 ?v2 \n" +
"WHERE \n" +
"{ \n" +
"\t?v0 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://dbpedia.org/class/yago/LanguagesOfBotswana> . \n" +
"\t?v0 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://dbpedia.org/class/yago/LanguagesOfNamibia> . \n" +
"\t?v0 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://dbpedia.org/ontology/Language> . \n" +
"} \n";
}
//if (example === "q7")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 WHERE \n" +
//"{ \n" +
//"\t<http://dbpedia.org/resource/Albedo> <http://dbpedia.org/ontology/wikiPageWikiLink> ?v0 .\n" +
//"\t?v1 <http://dbpedia.org/ontology/wikiPageWikiLink> ?v0 .\n" +
//"}\n";
//}
//if (example === "q8")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 ?v4 ?v5 ?v6 WHERE \n" +
//"{\n" +
//"\t?v0 <http://www.w3.org/2004/02/skos/core#broader> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/child> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/spouse> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/profession> ?v4 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/party> ?v5 .\n" +
//"\t?v0 <http://dbpedia.prg/ontology/militaryRank> ?v6 .\n" +
//"}\n";
//}
//if (example === "q9")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?V0 <http://dbpedia.org/ontology/location> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/typeOfElectrification> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Koltsevaya_Line> .\n" +
//"\t?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v3 .\n" +
//"}\n";
//}
//if (example === "q10")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?v0 <http://dbpedia.org/property/hasPhotoCollection> ?v1 .\n" +
//"\t?v0 <xmlns.com/foaf/0.1/depiction> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/abstract> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Combination> .\n" +
//"}\n";
//}
}

82
Server/web/freebase/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>Freebase Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>Freebase Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>Freebase Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="freebase" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<!--<option value="q4">q4</option>-->
<!--<option value="q5">q5</option>-->
<!--<option value="q6">q6</option>-->
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

242
Server/web/freebase/index.js Normal file
View File

@ -0,0 +1,242 @@
function query(dp) {
var encodeVal = escape(dp);
//alert(encodeVal);
var format = document.getElementById("element_5").value;
if(format == "1")
format = "html";
else if(format == "2")
format = "txt";
else if(format == "3")
format = "csv";
else if(format == "4")
format = "json";
var argu = "?operation=query&format=" + format + "&sparql=" + dp;
var encodeArgu = escape(argu);
if(format != "html")
{
/*
$.get(encodeArgu, function(data, status){
if(status == "success")
{
alert("success");
}
});
*/
var element = document.getElementById("hideLink");
element.setAttribute("href", encodeArgu);
/*
if(format == "txt")
element.setAttribute("download", "sparql.txt");
else if(format == "csv")
element.setAttribute("download", "sparql.csv");
else if(format == "json")
element.setAttribute("download", "sparql.json");
*/
if(/msie/i.test(navigator.userAgent))
{
element.fireEvent("onclick");
}
else
{
var e = document.createEvent("MouseEvents");
e.initEvent("click", true, true);
element.dispatchEvent(e);
}
}
else
{
$.get(encodeArgu, function(data, status){
if(status=="success"){
//toTxt();
//alert(data);
var parts = data.split("+");
var fileName = parts[2];
var lines = Number(parts[1]);
//alert(lines);
if(lines > 100)
lines = 100;
//alert(lines);
var items = parts[3].split("\n");
//alert(items[0]);
var valNum = items[0].split("?");
var rows = valNum.length - 1;
//alert(rows);
var page = '<html><div align="left"><a href="javascript:void(0);" id="back" style="font-size:16px;color:blue">Click to Return</a>';
page = page + '<a id="download" style="font-size:16px;margin-left:20px">Click to Download</a>';
page = page + '<a href="/" id="trick" style="display: none">Click to back</a>';
page = page + '<p>Total answers: ';
page = page + parts[1];
page = page + '</p>';
if(parts[0] == "1")
{
page = page + '<p>Number of answers is too large, show only 100 of them, click to download the full result!</p>';
}
page = page + '</div><table border="1" style = "font-size:medium">';
page = page + "<tr>";
for(var ii = 1; ii <= rows; ii++)
{
page = page + "<th>" + "?" + valNum[ii] + "</th>";
}
page = page + "</tr>";
var i = 1;
var j = 0;
for (i = 1; i <= lines; i++)
{
page = page + "<tr>";
//alert(items[i]);
var tmpItem = items[i].replace(/"([^"]*)"/g, "<$1>");
//alert(tmpItem);
var item = tmpItem.split(">");
//alert(item.length);
for(j = 0; j < rows; j++)
{
//alert(item[j]);
page = page + '<td>' + item[j].replace("<", "") + '</td>';
}
page = page + "</tr>";
}
page = page + '</table></div></html>';
$("#main_body").empty();
$("#main_body").append(page);
var tmp1 = "?operation=download&filepath=" + fileName;
var request1 = escape(tmp1);
var element1 = document.getElementById("download");
element1.setAttribute("href", request1);
element1.setAttribute("download", "sparql.txt");
var tmp2 = "?operation=delete&filepath=" + fileName;
var request2 = escape(tmp2);
var element2 = document.getElementById("back");
//!Notice: element2 is a "<a>" tag, and it has two actions, href and onclick, be careful with the executing order of these two actions.
//in this case, we use a call-back function to prevent strange things. we return to the origin web page after the request to delete file returns successfully.
element2.onclick = function(){
$.get(request2, function(data, status){
//alert("delete return");
var element3 = document.getElementById("trick");
if(/msie/i.test(navigator.userAgent))
{
element3.fireEvent("onclick");
}
else
{
var e1 = document.createEvent("MouseEvents");
e1.initEvent("click", true, true);
element3.dispatchEvent(e1);
}
});
}
}
});
}
}
function handleQueryExample()
{
var example = document.getElementById("example").value;
if (example === "q1")
{
document.getElementById("element_3").value = "select ?s where \n" +
"{ \n" +
"\t?s <http://www.w3.org/2000/01/rdf-schema#label> \"footballdb ID\"@en . \n" +
"\t?s <http://rdf.freebase.com/ns/type.property.schema> <http://rdf.freebase.com/ns/american_football.football_player> . \n" +
"\t?s <http://rdf.freebase.com/ns/type.property.unique> \"true\" . \n" +
"} \n";
}
if (example === "q2")
{
document.getElementById("element_3").value = "select ?s where \n" +
"{ \n" +
"\t?s <http://www.w3.org/2000/01/rdf-schema#label> \"footballdb ID\"@en . \n" +
"} \n";
}
if (example === "q3")
{
document.getElementById("element_3").value = "select ?s ?s2 where \n" +
"{ \n" +
"\t?s <http://www.w3.org/2000/01/rdf-schema#label> ?o . \n" +
"\t?s <http://rdf.freebase.com/ns/type.object.type> ?s2 . \n" +
"\t?s2 <http://www.w3.org/2000/01/rdf-schema#label> ?o . \n" +
"\t<http://rdf.freebase.com/ns/m.026wjqb> <http://rdf.freebase.com/ns/type.object.type> ?s2 . \n" +
"} \n";
}
//if (example === "q4")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:GraduateStudent>. \n" +
//"\t?y <rdf:type> <ub:University>. \n" +
//"\t?z <rdf:type> <ub:Department>. \n" +
//"\t?x <ub:memberOf> ?z. \n" +
//"\t?z <ub:subOrganizationOf> ?y. \n" +
//"\t?x <ub:undergraduateDegreeFrom> ?y. \n" +
//"} \n";
//}
//if (example === "q5")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:Course>. \n" +
//"\t?x <ub:name> ?y. \n" +
//"} \n";
//}
//if (example === "q6")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"\t?y <ub:name> <Course1>. \n" +
//"\t?x <ub:takesCourse> ?y. \n" +
//"\t?z <ub:teacherOf> ?y. \n" +
//"\t?z <ub:name> <FullProfessor1>. \n" +
//"\t?z <ub:worksFor> ?w. \n" +
//"\t?w <ub:name> <Department0>. \n" +
//"} \n";
//}
//if (example === "q7")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"}\n";
//}
//if (example === "q8")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 ?v4 ?v5 ?v6 WHERE \n" +
//"{\n" +
//"\t?v0 <http://www.w3.org/2004/02/skos/core#broader> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/child> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/spouse> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/profession> ?v4 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/party> ?v5 .\n" +
//"\t?v0 <http://dbpedia.prg/ontology/militaryRank> ?v6 .\n" +
//"}\n";
//}
//if (example === "q9")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?V0 <http://dbpedia.org/ontology/location> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/typeOfElectrification> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Koltsevaya_Line> .\n" +
//"\t?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v3 .\n" +
//"}\n";
//}
//if (example === "q10")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?v0 <http://dbpedia.org/property/hasPhotoCollection> ?v1 .\n" +
//"\t?v0 <xmlns.com/foaf/0.1/depiction> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/abstract> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Combination> .\n" +
//"}\n";
//}
}

82
Server/web/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>SPARQL Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>SPARQL Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>SPARQL Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="lubm" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<!--<option value="q4">q4</option>-->
<!--<option value="q5">q5</option>-->
<!--<option value="q6">q6</option>-->
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

234
Server/web/index.js Normal file
View File

@ -0,0 +1,234 @@
function query(dp) {
var encodeVal = escape(dp);
//alert(encodeVal);
var format = document.getElementById("element_5").value;
if(format == "1")
format = "html";
else if(format == "2")
format = "txt";
else if(format == "3")
format = "csv";
else if(format == "4")
format = "json";
var argu = "?operation=query&format=" + format + "&sparql=" + dp;
var encodeArgu = escape(argu);
if(format != "html")
{
/*
$.get(encodeArgu, function(data, status){
if(status == "success")
{
alert("success");
}
});
*/
var element = document.getElementById("hideLink");
element.setAttribute("href", encodeArgu);
/*
if(format == "txt")
element.setAttribute("download", "sparql.txt");
else if(format == "csv")
element.setAttribute("download", "sparql.csv");
else if(format == "json")
element.setAttribute("download", "sparql.json");
*/
if(/msie/i.test(navigator.userAgent))
{
element.fireEvent("onclick");
}
else
{
var e = document.createEvent("MouseEvents");
e.initEvent("click", true, true);
element.dispatchEvent(e);
}
}
else
{
$.get(encodeArgu, function(data, status){
if(status=="success"){
//toTxt();
//alert(data);
var parts = data.split("+");
var fileName = parts[2];
var lines = Number(parts[1]);
//alert(lines);
if(lines > 100)
lines = 100;
//alert(lines);
var items = parts[3].split("\n");
//alert(items[0]);
var valNum = items[0].split("?");
var rows = valNum.length - 1;
//alert(rows);
var page = '<html><div align="left"><a href="javascript:void(0);" id="back" style="font-size:16px;color:blue">Click to Return</a>';
page = page + '<a id="download" style="font-size:16px;margin-left:20px">Click to Download</a>';
page = page + '<a href="/" id="trick" style="display: none">Click to back</a>';
page = page + '<p>Total answers: ';
page = page + parts[1];
page = page + '</p>';
if(parts[0] == "1")
{
page = page + '<p>Number of answers is too large, show only 100 of them, click to download the full result!</p>';
}
page = page + '</div><table border="1" style = "font-size:medium">';
page = page + "<tr>";
for(var ii = 1; ii <= rows; ii++)
{
page = page + "<th>" + "?" + valNum[ii] + "</th>";
}
page = page + "</tr>";
var i = 1;
var j = 0;
for (i = 1; i <= lines; i++)
{
page = page + "<tr>";
//alert(items[i]);
var tmpItem = items[i].replace(/"([^"]*)"/g, "<$1>");
//alert(tmpItem);
var item = tmpItem.split(">");
//alert(item.length);
for(j = 0; j < rows; j++)
{
//alert(item[j]);
page = page + '<td>' + item[j].replace("<", "") + '</td>';
}
page = page + "</tr>";
}
page = page + '</table></div></html>';
$("#main_body").empty();
$("#main_body").append(page);
var tmp1 = "?operation=download&filepath=" + fileName;
var request1 = escape(tmp1);
var element1 = document.getElementById("download");
element1.setAttribute("href", request1);
element1.setAttribute("download", "sparql.txt");
var tmp2 = "?operation=delete&filepath=" + fileName;
var request2 = escape(tmp2);
var element2 = document.getElementById("back");
//!Notice: element2 is a "<a>" tag, and it has two actions, href and onclick, be careful with the executing order of these two actions.
//in this case, we use a call-back function to prevent strange things. we return to the origin web page after the request to delete file returns successfully.
element2.onclick = function(){
$.get(request2, function(data, status){
//alert("delete return");
var element3 = document.getElementById("trick");
if(/msie/i.test(navigator.userAgent))
{
element3.fireEvent("onclick");
}
else
{
var e1 = document.createEvent("MouseEvents");
e1.initEvent("click", true, true);
element3.dispatchEvent(e1);
}
});
}
}
});
}
}
function handleQueryExample()
{
var example = document.getElementById("example").value;
if (example === "q1")
{
document.getElementById("element_3").value = "SELECT DISTINCT ?p \n" +
"WHERE \n" +
"{ \n" +
"\t?s ?p ?o . \n" +
"} \n";
}
if (example === "q2")
{
document.getElementById("element_3").value = "SELECT ?x WHERE \n" +
"{ \n" +
"\t?x <ub:name> <FullProfessor0> .\n" +
"} \n";
}
if (example === "q3")
{
document.getElementById("element_3").value = "SELECT ?x WHERE \n" +
"{ \n" +
"\t?x <rdf:type> <ub:Course> .\n" +
"\t?x <ub:name> ?y .\n" +
"} \n";
}
//if (example === "q4")
//{
//document.getElementById("element_3").value = "SELECT DISTINCT ?v0 WHERE \n" +
//"{ \n" +
//"\t?v0 <http://purl.org/dc/terms/subject> <http://dbpedia.org/resource/Category:956_births> .\n" +
//"\t{?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v1 .}\n" + "UNION\n" +
//"\t{?v0 <http://dbpedia.org/prpperty/hasPhotoCollection> ?v2 .}\n" +
//"} \n";
//}
//if (example === "q5")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v2 ?v3 WHERE \n" +
//"{ \n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Autism> .\n" +
//"\t?v2 <http://dbpedia.org/property/candidate> ?v0 .\n" +
//"\t?v3 <http://dbpedia.org/property/constituencyWestminster> ?v2 .\n" +
//"} \n";
//}
//if (example === "q6")
//{
//document.getElementById("element_3").value = "SELECT ?V1 ?V2 WHERE \n" +
//"{ \n" +
//"\t?v1 <http://www.w3.org/2002/07/owl#sameAs> <http://it.dbpedia.org/resource/Category:Filosofi_del_IV_secolo_a.C.> .\n" +
//"\t?v2 <dbpedia.org/ontology/wikiPageWikiLink> ?v1 .\n" +
//"\t<http://dbpedia.org/resource/Chinese_classics> <http://dbpedia.org/ontology/wikiPageWikiLink> ?v2 .\n" +
//"} \n";
//}
//if (example === "q7")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 WHERE \n" +
//"{ \n" +
//"\t<http://dbpedia.org/resource/Albedo> <http://dbpedia.org/ontology/wikiPageWikiLink> ?v0 .\n" +
//"\t?v1 <http://dbpedia.org/ontology/wikiPageWikiLink> ?v0 .\n" +
//"}\n";
//}
//if (example === "q8")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 ?v4 ?v5 ?v6 WHERE \n" +
//"{\n" +
//"\t?v0 <http://www.w3.org/2004/02/skos/core#broader> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/child> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/spouse> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/profession> ?v4 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/party> ?v5 .\n" +
//"\t?v0 <http://dbpedia.prg/ontology/militaryRank> ?v6 .\n" +
//"}\n";
//}
//if (example === "q9")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?V0 <http://dbpedia.org/ontology/location> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/typeOfElectrification> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Koltsevaya_Line> .\n" +
//"\t?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v3 .\n" +
//"}\n";
//}
//if (example === "q10")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?v0 <http://dbpedia.org/property/hasPhotoCollection> ?v1 .\n" +
//"\t?v0 <xmlns.com/foaf/0.1/depiction> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/abstract> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Combination> .\n" +
//"}\n";
//}
}

67
Server/web/openkg.html Normal file
View File

@ -0,0 +1,67 @@
<!--=============================================================================-->
<!--Filename: openkg.html-->
<!--Author: lijing-->
<!--Mail: li_jing@pku.edu.cn-->
<!--Last Modified: 2017-07-26 11:09-->
<!--Description: written by lijing-->
<!--=============================================================================-->
<!--USAGE: start a web server like Apache, and place this file as the server's default page-->
<!--Remember to rename this file as index.html-->
<!--Then we can use URL like openkg.gstore-pku.com to visit this page, and via this page we can link to other real openkg databases-->
<!--TODO: transform json or other formats to RDF N3 format in openkg-->
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="description" content="">
<meta name="HandheldFriendly" content="True">
<meta name="MobileOptimized" content="320">
<meta name="viewport" content="initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no">
<title>OpenKG datasets</title>
<link rel="alternate" type="application/rss+xml" title="" href="feed/index.html">
<link href="http://fonts.googleapis.com/css?family=Raleway:700,300" rel="stylesheet"
type="text/css">
<link rel="stylesheet" href="style.css">
<!--<link rel="stylesheet" href="css/prettify.css">-->
</head>
<body>
<div class="wrapper">
<header>
<div class="container">
<h2 class="lone-header">OpenKG datasets</h2>
</div>
</header>
<section>
<div class="container">
<ul class="docs-nav">
<li><strong>Tourist</strong></li>
<li><a href="http://tourist.gstore-pku.com" class="cc-active">tourist.gstore-pku.com</a></li>
<li><strong>Music</strong></li>
<li><a href="http://music.gstore-pku.com" class="cc-active">music.gstore-pku.com</a></li>
<li><strong>Emergency</strong></li>
<li><a href="http://emergency.gstore-pku.com" class="cc-active">emergency.gstore-pku.com</a></li>
<li><strong>Breast Cancer</strong></li>
<li><a href="http://breastcancer0.gstore-pku.com" class="cc-active">breastcancer0.gstore-pku.com</a></li>
<li><a href="http://breastcancer1.gstore-pku.com" class="cc-active">breastcancer1.gstore-pku.com</a></li>
<li><a href="http://breastcancer2.gstore-pku.com" class="cc-active">breastcancer2.gstore-pku.com</a></li>
<li><a href="http://breastcancer3.gstore-pku.com" class="cc-active">breastcancer3.gstore-pku.com</a></li>
<li><a href="http://breastcancer4.gstore-pku.com" class="cc-active">breastcancer4.gstore-pku.com</a></li>
<li><strong>Clinga</strong></li>
<li><a href="http://clinga.gstore-pku.com" class="cc-active">clinga.gstore-pku.com</a></li>
<!--<li><strong>Person</strong></li>
<li><a href="http://person-zhishime.gstore-pku.com" class="cc-active">person-zhishime.gstore-pku.com</a></li>
<li><a href="http://person-multype.gstore-pku.com" class="cc-active">person-multype.gstore-pku.com</a></li>
<li><a href="http://person-taxonomy.gstore-pku.com" class="cc-active">person-taxonomy.gstore-pku.com</a></li>
-->
</ul>
</div>
</section>
</div>
<!--
<script src="https://cdn.bootcss.com/prettify/r298/run_prettify.js"></script>
<script src="js/layout.js"></script>-->
</body>
</html>

82
Server/web/openkg/breastcancer0/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>OpenKG Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>OpenKG Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>OpenKG Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="breast cancer 0" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<!--<option value="q4">q4</option>-->
<!--<option value="q5">q5</option>-->
<!--<option value="q6">q6</option>-->
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

241
Server/web/openkg/breastcancer0/index.js Normal file
View File

@ -0,0 +1,241 @@
function query(dp) {
var encodeVal = escape(dp);
//alert(encodeVal);
var format = document.getElementById("element_5").value;
if(format == "1")
format = "html";
else if(format == "2")
format = "txt";
else if(format == "3")
format = "csv";
else if(format == "4")
format = "json";
var argu = "?operation=query&format=" + format + "&sparql=" + dp;
var encodeArgu = escape(argu);
if(format != "html")
{
/*
$.get(encodeArgu, function(data, status){
if(status == "success")
{
alert("success");
}
});
*/
var element = document.getElementById("hideLink");
element.setAttribute("href", encodeArgu);
/*
if(format == "txt")
element.setAttribute("download", "sparql.txt");
else if(format == "csv")
element.setAttribute("download", "sparql.csv");
else if(format == "json")
element.setAttribute("download", "sparql.json");
*/
if(/msie/i.test(navigator.userAgent))
{
element.fireEvent("onclick");
}
else
{
var e = document.createEvent("MouseEvents");
e.initEvent("click", true, true);
element.dispatchEvent(e);
}
}
else
{
$.get(encodeArgu, function(data, status){
if(status=="success"){
//toTxt();
//alert(data);
var parts = data.split("+");
var fileName = parts[2];
var lines = Number(parts[1]);
//alert(lines);
if(lines > 100)
lines = 100;
//alert(lines);
var items = parts[3].split("\n");
//alert(items[0]);
var valNum = items[0].split("?");
var rows = valNum.length - 1;
//alert(rows);
var page = '<html><div align="left"><a href="javascript:void(0);" id="back" style="font-size:16px;color:blue">Click to Return</a>';
page = page + '<a id="download" style="font-size:16px;margin-left:20px">Click to Download</a>';
page = page + '<a href="/" id="trick" style="display: none">Click to back</a>';
page = page + '<p>Total answers: ';
page = page + parts[1];
page = page + '</p>';
if(parts[0] == "1")
{
page = page + '<p>Number of answers is too large, show only 100 of them, click to download the full result!</p>';
}
page = page + '</div><table border="1" style = "font-size:medium">';
page = page + "<tr>";
for(var ii = 1; ii <= rows; ii++)
{
page = page + "<th>" + "?" + valNum[ii] + "</th>";
}
page = page + "</tr>";
var i = 1;
var j = 0;
for (i = 1; i <= lines; i++)
{
page = page + "<tr>";
//alert(items[i]);
var tmpItem = items[i].replace(/"([^"]*)"/g, "<$1>");
//alert(tmpItem);
var item = tmpItem.split(">");
//alert(item.length);
for(j = 0; j < rows; j++)
{
//alert(item[j]);
page = page + '<td>' + item[j].replace("<", "") + '</td>';
}
page = page + "</tr>";
}
page = page + '</table></div></html>';
$("#main_body").empty();
$("#main_body").append(page);
var tmp1 = "?operation=download&filepath=" + fileName;
var request1 = escape(tmp1);
var element1 = document.getElementById("download");
element1.setAttribute("href", request1);
element1.setAttribute("download", "sparql.txt");
var tmp2 = "?operation=delete&filepath=" + fileName;
var request2 = escape(tmp2);
var element2 = document.getElementById("back");
//!Notice: element2 is a "<a>" tag, and it has two actions, href and onclick, be careful with the executing order of these two actions.
//in this case, we use a call-back function to prevent strange things. we return to the origin web page after the request to delete file returns successfully.
element2.onclick = function(){
$.get(request2, function(data, status){
//alert("delete return");
var element3 = document.getElementById("trick");
if(/msie/i.test(navigator.userAgent))
{
element3.fireEvent("onclick");
}
else
{
var e1 = document.createEvent("MouseEvents");
e1.initEvent("click", true, true);
element3.dispatchEvent(e1);
}
});
}
}
});
}
}
function handleQueryExample()
{
var example = document.getElementById("example").value;
if(example === "q1")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t?x <http://eurecaproject.eu/ctec/hasSenses> ?y .\n" +
"}\n";
}
if(example === "q2")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t?s <http://eurecaproject.eu/ctec/hasSenses> ?x .\n" +
"\t?x <http://eurecaproject.eu/ctec/Source> ?y .\n" +
"} \n";
}
if(example === "q3")
{
document.getElementById("element_3").value = "select ?x ?y \n" +
"where\n" +
"{ \n" +
"\t?s <http://eurecaproject.eu/ctec/hasAnnotation> ?o .\n" +
"\t?o <http://eurecaproject.eu/ctec/hasSenses> ?x.\n" +
"\t?x <http://eurecaproject.eu/ctec/Sense> ?y.\n" +
"} \n";
}
//if (example === "q4")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:GraduateStudent>. \n" +
//"\t?y <rdf:type> <ub:University>. \n" +
//"\t?z <rdf:type> <ub:Department>. \n" +
//"\t?x <ub:memberOf> ?z. \n" +
//"\t?z <ub:subOrganizationOf> ?y. \n" +
//"\t?x <ub:undergraduateDegreeFrom> ?y. \n" +
//"} \n";
//}
//if (example === "q5")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:Course>. \n" +
//"\t?x <ub:name> ?y. \n" +
//"} \n";
//}
//if (example === "q6")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"\t?y <ub:name> <Course1>. \n" +
//"\t?x <ub:takesCourse> ?y. \n" +
//"\t?z <ub:teacherOf> ?y. \n" +
//"\t?z <ub:name> <FullProfessor1>. \n" +
//"\t?z <ub:worksFor> ?w. \n" +
//"\t?w <ub:name> <Department0>. \n" +
//"} \n";
//}
//if (example === "q7")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"}\n";
//}
//if (example === "q8")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 ?v4 ?v5 ?v6 WHERE \n" +
//"{\n" +
//"\t?v0 <http://www.w3.org/2004/02/skos/core#broader> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/child> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/spouse> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/profession> ?v4 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/party> ?v5 .\n" +
//"\t?v0 <http://dbpedia.prg/ontology/militaryRank> ?v6 .\n" +
//"}\n";
//}
//if (example === "q9")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?V0 <http://dbpedia.org/ontology/location> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/typeOfElectrification> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Koltsevaya_Line> .\n" +
//"\t?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v3 .\n" +
//"}\n";
//}
//if (example === "q10")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?v0 <http://dbpedia.org/property/hasPhotoCollection> ?v1 .\n" +
//"\t?v0 <xmlns.com/foaf/0.1/depiction> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/abstract> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Combination> .\n" +
//"}\n";
//}
}

82
Server/web/openkg/breastcancer1/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>OpenKG Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>OpenKG Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>OpenKG Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="breast cancer 1" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<!--<option value="q4">q4</option>-->
<!--<option value="q5">q5</option>-->
<!--<option value="q6">q6</option>-->
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

241
Server/web/openkg/breastcancer1/index.js Normal file
View File

@ -0,0 +1,241 @@
function query(dp) {
var encodeVal = escape(dp);
//alert(encodeVal);
var format = document.getElementById("element_5").value;
if(format == "1")
format = "html";
else if(format == "2")
format = "txt";
else if(format == "3")
format = "csv";
else if(format == "4")
format = "json";
var argu = "?operation=query&format=" + format + "&sparql=" + dp;
var encodeArgu = escape(argu);
if(format != "html")
{
/*
$.get(encodeArgu, function(data, status){
if(status == "success")
{
alert("success");
}
});
*/
var element = document.getElementById("hideLink");
element.setAttribute("href", encodeArgu);
/*
if(format == "txt")
element.setAttribute("download", "sparql.txt");
else if(format == "csv")
element.setAttribute("download", "sparql.csv");
else if(format == "json")
element.setAttribute("download", "sparql.json");
*/
if(/msie/i.test(navigator.userAgent))
{
element.fireEvent("onclick");
}
else
{
var e = document.createEvent("MouseEvents");
e.initEvent("click", true, true);
element.dispatchEvent(e);
}
}
else
{
$.get(encodeArgu, function(data, status){
if(status=="success"){
//toTxt();
//alert(data);
var parts = data.split("+");
var fileName = parts[2];
var lines = Number(parts[1]);
//alert(lines);
if(lines > 100)
lines = 100;
//alert(lines);
var items = parts[3].split("\n");
//alert(items[0]);
var valNum = items[0].split("?");
var rows = valNum.length - 1;
//alert(rows);
var page = '<html><div align="left"><a href="javascript:void(0);" id="back" style="font-size:16px;color:blue">Click to Return</a>';
page = page + '<a id="download" style="font-size:16px;margin-left:20px">Click to Download</a>';
page = page + '<a href="/" id="trick" style="display: none">Click to back</a>';
page = page + '<p>Total answers: ';
page = page + parts[1];
page = page + '</p>';
if(parts[0] == "1")
{
page = page + '<p>Number of answers is too large, show only 100 of them, click to download the full result!</p>';
}
page = page + '</div><table border="1" style = "font-size:medium">';
page = page + "<tr>";
for(var ii = 1; ii <= rows; ii++)
{
page = page + "<th>" + "?" + valNum[ii] + "</th>";
}
page = page + "</tr>";
var i = 1;
var j = 0;
for (i = 1; i <= lines; i++)
{
page = page + "<tr>";
//alert(items[i]);
var tmpItem = items[i].replace(/"([^"]*)"/g, "<$1>");
//alert(tmpItem);
var item = tmpItem.split(">");
//alert(item.length);
for(j = 0; j < rows; j++)
{
//alert(item[j]);
page = page + '<td>' + item[j].replace("<", "") + '</td>';
}
page = page + "</tr>";
}
page = page + '</table></div></html>';
$("#main_body").empty();
$("#main_body").append(page);
var tmp1 = "?operation=download&filepath=" + fileName;
var request1 = escape(tmp1);
var element1 = document.getElementById("download");
element1.setAttribute("href", request1);
element1.setAttribute("download", "sparql.txt");
var tmp2 = "?operation=delete&filepath=" + fileName;
var request2 = escape(tmp2);
var element2 = document.getElementById("back");
//!Notice: element2 is a "<a>" tag, and it has two actions, href and onclick, be careful with the executing order of these two actions.
//in this case, we use a call-back function to prevent strange things. we return to the origin web page after the request to delete file returns successfully.
element2.onclick = function(){
$.get(request2, function(data, status){
//alert("delete return");
var element3 = document.getElementById("trick");
if(/msie/i.test(navigator.userAgent))
{
element3.fireEvent("onclick");
}
else
{
var e1 = document.createEvent("MouseEvents");
e1.initEvent("click", true, true);
element3.dispatchEvent(e1);
}
});
}
}
});
}
}
function handleQueryExample()
{
var example = document.getElementById("example").value;
if(example === "q1")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t?x <http://eurecaproject.eu/ctec/hasText> ?y.\n" +
"}\n";
}
if(example === "q2")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t?x <http://eurecaproject.eu/ctec/hasFragment> ?z .\n" +
"\t?z <http://eurecaproject.eu/ctec/isA> ?y .\n" +
"} \n";
}
if(example === "q3")
{
document.getElementById("element_3").value = "select ?x ?y \n" +
"where\n" +
"{ \n" +
"\t?s <http://eurecaproject.eu/ctec/hasFragment> ?z .\n" +
"\t?z <http://eurecaproject.eu/ctec/isA> ?x .\n" +
"\t?z <http://eurecaproject.eu/ctec/hasTerm> ?y .\n" +
"} \n";
}
//if (example === "q4")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:GraduateStudent>. \n" +
//"\t?y <rdf:type> <ub:University>. \n" +
//"\t?z <rdf:type> <ub:Department>. \n" +
//"\t?x <ub:memberOf> ?z. \n" +
//"\t?z <ub:subOrganizationOf> ?y. \n" +
//"\t?x <ub:undergraduateDegreeFrom> ?y. \n" +
//"} \n";
//}
//if (example === "q5")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:Course>. \n" +
//"\t?x <ub:name> ?y. \n" +
//"} \n";
//}
//if (example === "q6")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"\t?y <ub:name> <Course1>. \n" +
//"\t?x <ub:takesCourse> ?y. \n" +
//"\t?z <ub:teacherOf> ?y. \n" +
//"\t?z <ub:name> <FullProfessor1>. \n" +
//"\t?z <ub:worksFor> ?w. \n" +
//"\t?w <ub:name> <Department0>. \n" +
//"} \n";
//}
//if (example === "q7")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"}\n";
//}
//if (example === "q8")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 ?v4 ?v5 ?v6 WHERE \n" +
//"{\n" +
//"\t?v0 <http://www.w3.org/2004/02/skos/core#broader> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/child> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/spouse> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/profession> ?v4 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/party> ?v5 .\n" +
//"\t?v0 <http://dbpedia.prg/ontology/militaryRank> ?v6 .\n" +
//"}\n";
//}
//if (example === "q9")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?V0 <http://dbpedia.org/ontology/location> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/typeOfElectrification> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Koltsevaya_Line> .\n" +
//"\t?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v3 .\n" +
//"}\n";
//}
//if (example === "q10")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?v0 <http://dbpedia.org/property/hasPhotoCollection> ?v1 .\n" +
//"\t?v0 <xmlns.com/foaf/0.1/depiction> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/abstract> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Combination> .\n" +
//"}\n";
//}
}

82
Server/web/openkg/breastcancer2/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>OpenKG Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>OpenKG Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>OpenKG Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="breast cancer 2" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<!--<option value="q4">q4</option>-->
<!--<option value="q5">q5</option>-->
<!--<option value="q6">q6</option>-->
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

241
Server/web/openkg/breastcancer2/index.js Normal file
View File

@ -0,0 +1,241 @@
function query(dp) {
var encodeVal = escape(dp);
//alert(encodeVal);
var format = document.getElementById("element_5").value;
if(format == "1")
format = "html";
else if(format == "2")
format = "txt";
else if(format == "3")
format = "csv";
else if(format == "4")
format = "json";
var argu = "?operation=query&format=" + format + "&sparql=" + dp;
var encodeArgu = escape(argu);
if(format != "html")
{
/*
$.get(encodeArgu, function(data, status){
if(status == "success")
{
alert("success");
}
});
*/
var element = document.getElementById("hideLink");
element.setAttribute("href", encodeArgu);
/*
if(format == "txt")
element.setAttribute("download", "sparql.txt");
else if(format == "csv")
element.setAttribute("download", "sparql.csv");
else if(format == "json")
element.setAttribute("download", "sparql.json");
*/
if(/msie/i.test(navigator.userAgent))
{
element.fireEvent("onclick");
}
else
{
var e = document.createEvent("MouseEvents");
e.initEvent("click", true, true);
element.dispatchEvent(e);
}
}
else
{
$.get(encodeArgu, function(data, status){
if(status=="success"){
//toTxt();
//alert(data);
var parts = data.split("+");
var fileName = parts[2];
var lines = Number(parts[1]);
//alert(lines);
if(lines > 100)
lines = 100;
//alert(lines);
var items = parts[3].split("\n");
//alert(items[0]);
var valNum = items[0].split("?");
var rows = valNum.length - 1;
//alert(rows);
var page = '<html><div align="left"><a href="javascript:void(0);" id="back" style="font-size:16px;color:blue">Click to Return</a>';
page = page + '<a id="download" style="font-size:16px;margin-left:20px">Click to Download</a>';
page = page + '<a href="/" id="trick" style="display: none">Click to back</a>';
page = page + '<p>Total answers: ';
page = page + parts[1];
page = page + '</p>';
if(parts[0] == "1")
{
page = page + '<p>Number of answers is too large, show only 100 of them, click to download the full result!</p>';
}
page = page + '</div><table border="1" style = "font-size:medium">';
page = page + "<tr>";
for(var ii = 1; ii <= rows; ii++)
{
page = page + "<th>" + "?" + valNum[ii] + "</th>";
}
page = page + "</tr>";
var i = 1;
var j = 0;
for (i = 1; i <= lines; i++)
{
page = page + "<tr>";
//alert(items[i]);
var tmpItem = items[i].replace(/"([^"]*)"/g, "<$1>");
//alert(tmpItem);
var item = tmpItem.split(">");
//alert(item.length);
for(j = 0; j < rows; j++)
{
//alert(item[j]);
page = page + '<td>' + item[j].replace("<", "") + '</td>';
}
page = page + "</tr>";
}
page = page + '</table></div></html>';
$("#main_body").empty();
$("#main_body").append(page);
var tmp1 = "?operation=download&filepath=" + fileName;
var request1 = escape(tmp1);
var element1 = document.getElementById("download");
element1.setAttribute("href", request1);
element1.setAttribute("download", "sparql.txt");
var tmp2 = "?operation=delete&filepath=" + fileName;
var request2 = escape(tmp2);
var element2 = document.getElementById("back");
//!Notice: element2 is a "<a>" tag, and it has two actions, href and onclick, be careful with the executing order of these two actions.
//in this case, we use a call-back function to prevent strange things. we return to the origin web page after the request to delete file returns successfully.
element2.onclick = function(){
$.get(request2, function(data, status){
//alert("delete return");
var element3 = document.getElementById("trick");
if(/msie/i.test(navigator.userAgent))
{
element3.fireEvent("onclick");
}
else
{
var e1 = document.createEvent("MouseEvents");
e1.initEvent("click", true, true);
element3.dispatchEvent(e1);
}
});
}
}
});
}
}
function handleQueryExample()
{
var example = document.getElementById("example").value;
if(example === "q1")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t?x <http://linkedlifedata.com/resource/pubmed/hasArticleTitle> ?y .\n" +
"}\n";
}
if(example === "q2")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t?x <http://linkedlifedata.com/resource/pubmed/hasAuthor> ?z .\n" +
"\t?z <http://linkedlifedata.com/resource/pubmed/hasFullName> ?y .\n" +
"} \n";
}
if(example === "q3")
{
document.getElementById("element_3").value = "select ?x ?y \n" +
"where\n" +
"{ \n" +
"\t?s <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> ?x .\n" +
"\t?s <http://linkedlifedata.com/resource/pubmed/hasChemical> ?z .\n" +
"\t?z <http://www.w3.org/2000/01/rdf-schema#label> ?y .\n" +
"} \n";
}
//if (example === "q4")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:GraduateStudent>. \n" +
//"\t?y <rdf:type> <ub:University>. \n" +
//"\t?z <rdf:type> <ub:Department>. \n" +
//"\t?x <ub:memberOf> ?z. \n" +
//"\t?z <ub:subOrganizationOf> ?y. \n" +
//"\t?x <ub:undergraduateDegreeFrom> ?y. \n" +
//"} \n";
//}
//if (example === "q5")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:Course>. \n" +
//"\t?x <ub:name> ?y. \n" +
//"} \n";
//}
//if (example === "q6")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"\t?y <ub:name> <Course1>. \n" +
//"\t?x <ub:takesCourse> ?y. \n" +
//"\t?z <ub:teacherOf> ?y. \n" +
//"\t?z <ub:name> <FullProfessor1>. \n" +
//"\t?z <ub:worksFor> ?w. \n" +
//"\t?w <ub:name> <Department0>. \n" +
//"} \n";
//}
//if (example === "q7")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"}\n";
//}
//if (example === "q8")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 ?v4 ?v5 ?v6 WHERE \n" +
//"{\n" +
//"\t?v0 <http://www.w3.org/2004/02/skos/core#broader> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/child> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/spouse> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/profession> ?v4 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/party> ?v5 .\n" +
//"\t?v0 <http://dbpedia.prg/ontology/militaryRank> ?v6 .\n" +
//"}\n";
//}
//if (example === "q9")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?V0 <http://dbpedia.org/ontology/location> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/typeOfElectrification> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Koltsevaya_Line> .\n" +
//"\t?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v3 .\n" +
//"}\n";
//}
//if (example === "q10")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?v0 <http://dbpedia.org/property/hasPhotoCollection> ?v1 .\n" +
//"\t?v0 <xmlns.com/foaf/0.1/depiction> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/abstract> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Combination> .\n" +
//"}\n";
//}
}

82
Server/web/openkg/breastcancer3/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>OpenKG Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>OpenKG Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>OpenKG Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="breast cancer 3" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<!--<option value="q4">q4</option>-->
<!--<option value="q5">q5</option>-->
<!--<option value="q6">q6</option>-->
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

241
Server/web/openkg/breastcancer3/index.js Normal file
View File

@ -0,0 +1,241 @@
function query(dp) {
var encodeVal = escape(dp);
//alert(encodeVal);
var format = document.getElementById("element_5").value;
if(format == "1")
format = "html";
else if(format == "2")
format = "txt";
else if(format == "3")
format = "csv";
else if(format == "4")
format = "json";
var argu = "?operation=query&format=" + format + "&sparql=" + dp;
var encodeArgu = escape(argu);
if(format != "html")
{
/*
$.get(encodeArgu, function(data, status){
if(status == "success")
{
alert("success");
}
});
*/
var element = document.getElementById("hideLink");
element.setAttribute("href", encodeArgu);
/*
if(format == "txt")
element.setAttribute("download", "sparql.txt");
else if(format == "csv")
element.setAttribute("download", "sparql.csv");
else if(format == "json")
element.setAttribute("download", "sparql.json");
*/
if(/msie/i.test(navigator.userAgent))
{
element.fireEvent("onclick");
}
else
{
var e = document.createEvent("MouseEvents");
e.initEvent("click", true, true);
element.dispatchEvent(e);
}
}
else
{
$.get(encodeArgu, function(data, status){
if(status=="success"){
//toTxt();
//alert(data);
var parts = data.split("+");
var fileName = parts[2];
var lines = Number(parts[1]);
//alert(lines);
if(lines > 100)
lines = 100;
//alert(lines);
var items = parts[3].split("\n");
//alert(items[0]);
var valNum = items[0].split("?");
var rows = valNum.length - 1;
//alert(rows);
var page = '<html><div align="left"><a href="javascript:void(0);" id="back" style="font-size:16px;color:blue">Click to Return</a>';
page = page + '<a id="download" style="font-size:16px;margin-left:20px">Click to Download</a>';
page = page + '<a href="/" id="trick" style="display: none">Click to back</a>';
page = page + '<p>Total answers: ';
page = page + parts[1];
page = page + '</p>';
if(parts[0] == "1")
{
page = page + '<p>Number of answers is too large, show only 100 of them, click to download the full result!</p>';
}
page = page + '</div><table border="1" style = "font-size:medium">';
page = page + "<tr>";
for(var ii = 1; ii <= rows; ii++)
{
page = page + "<th>" + "?" + valNum[ii] + "</th>";
}
page = page + "</tr>";
var i = 1;
var j = 0;
for (i = 1; i <= lines; i++)
{
page = page + "<tr>";
//alert(items[i]);
var tmpItem = items[i].replace(/"([^"]*)"/g, "<$1>");
//alert(tmpItem);
var item = tmpItem.split(">");
//alert(item.length);
for(j = 0; j < rows; j++)
{
//alert(item[j]);
page = page + '<td>' + item[j].replace("<", "") + '</td>';
}
page = page + "</tr>";
}
page = page + '</table></div></html>';
$("#main_body").empty();
$("#main_body").append(page);
var tmp1 = "?operation=download&filepath=" + fileName;
var request1 = escape(tmp1);
var element1 = document.getElementById("download");
element1.setAttribute("href", request1);
element1.setAttribute("download", "sparql.txt");
var tmp2 = "?operation=delete&filepath=" + fileName;
var request2 = escape(tmp2);
var element2 = document.getElementById("back");
//!Notice: element2 is a "<a>" tag, and it has two actions, href and onclick, be careful with the executing order of these two actions.
//in this case, we use a call-back function to prevent strange things. we return to the origin web page after the request to delete file returns successfully.
element2.onclick = function(){
$.get(request2, function(data, status){
//alert("delete return");
var element3 = document.getElementById("trick");
if(/msie/i.test(navigator.userAgent))
{
element3.fireEvent("onclick");
}
else
{
var e1 = document.createEvent("MouseEvents");
e1.initEvent("click", true, true);
element3.dispatchEvent(e1);
}
});
}
}
});
}
}
function handleQueryExample()
{
var example = document.getElementById("example").value;
if(example === "q1")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t<http://wasp.cs.vu.nl/sct/id#NCT00000529> ?x ?y .\n" +
"}\n";
}
if(example === "q2")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t?s <http://wasp.cs.vu.nl/sct/sct#BriefTitle> ?x .\n" +
"\t?s <http://wasp.cs.vu.nl/sct/sct#LeadSponsorAgency> ?y .\n" +
"} \n";
}
if(example === "q3")
{
document.getElementById("element_3").value = "select ?x ?y \n" +
"where\n" +
"{ \n" +
"\t?s <http://wasp.cs.vu.nl/sct/sct#BriefTitle> ?x .\n" +
"\t?s <http://wasp.cs.vu.nl/sct/sct#Reference> ?z .\n" +
"\t?z <http://wasp.cs.vu.nl/sct/sct#PMID> ?y .\n" +
"} \n";
}
//if (example === "q4")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:GraduateStudent>. \n" +
//"\t?y <rdf:type> <ub:University>. \n" +
//"\t?z <rdf:type> <ub:Department>. \n" +
//"\t?x <ub:memberOf> ?z. \n" +
//"\t?z <ub:subOrganizationOf> ?y. \n" +
//"\t?x <ub:undergraduateDegreeFrom> ?y. \n" +
//"} \n";
//}
//if (example === "q5")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:Course>. \n" +
//"\t?x <ub:name> ?y. \n" +
//"} \n";
//}
//if (example === "q6")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"\t?y <ub:name> <Course1>. \n" +
//"\t?x <ub:takesCourse> ?y. \n" +
//"\t?z <ub:teacherOf> ?y. \n" +
//"\t?z <ub:name> <FullProfessor1>. \n" +
//"\t?z <ub:worksFor> ?w. \n" +
//"\t?w <ub:name> <Department0>. \n" +
//"} \n";
//}
//if (example === "q7")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"}\n";
//}
//if (example === "q8")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 ?v4 ?v5 ?v6 WHERE \n" +
//"{\n" +
//"\t?v0 <http://www.w3.org/2004/02/skos/core#broader> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/child> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/spouse> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/profession> ?v4 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/party> ?v5 .\n" +
//"\t?v0 <http://dbpedia.prg/ontology/militaryRank> ?v6 .\n" +
//"}\n";
//}
//if (example === "q9")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?V0 <http://dbpedia.org/ontology/location> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/typeOfElectrification> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Koltsevaya_Line> .\n" +
//"\t?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v3 .\n" +
//"}\n";
//}
//if (example === "q10")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?v0 <http://dbpedia.org/property/hasPhotoCollection> ?v1 .\n" +
//"\t?v0 <xmlns.com/foaf/0.1/depiction> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/abstract> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Combination> .\n" +
//"}\n";
//}
}

82
Server/web/openkg/breastcancer4/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>OpenKG Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>OpenKG Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>OpenKG Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="breast cancer 4" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<!--<option value="q4">q4</option>-->
<!--<option value="q5">q5</option>-->
<!--<option value="q6">q6</option>-->
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

241
Server/web/openkg/breastcancer4/index.js Normal file
View File

@ -0,0 +1,241 @@
function query(dp) {
var encodeVal = escape(dp);
//alert(encodeVal);
var format = document.getElementById("element_5").value;
if(format == "1")
format = "html";
else if(format == "2")
format = "txt";
else if(format == "3")
format = "csv";
else if(format == "4")
format = "json";
var argu = "?operation=query&format=" + format + "&sparql=" + dp;
var encodeArgu = escape(argu);
if(format != "html")
{
/*
$.get(encodeArgu, function(data, status){
if(status == "success")
{
alert("success");
}
});
*/
var element = document.getElementById("hideLink");
element.setAttribute("href", encodeArgu);
/*
if(format == "txt")
element.setAttribute("download", "sparql.txt");
else if(format == "csv")
element.setAttribute("download", "sparql.csv");
else if(format == "json")
element.setAttribute("download", "sparql.json");
*/
if(/msie/i.test(navigator.userAgent))
{
element.fireEvent("onclick");
}
else
{
var e = document.createEvent("MouseEvents");
e.initEvent("click", true, true);
element.dispatchEvent(e);
}
}
else
{
$.get(encodeArgu, function(data, status){
if(status=="success"){
//toTxt();
//alert(data);
var parts = data.split("+");
var fileName = parts[2];
var lines = Number(parts[1]);
//alert(lines);
if(lines > 100)
lines = 100;
//alert(lines);
var items = parts[3].split("\n");
//alert(items[0]);
var valNum = items[0].split("?");
var rows = valNum.length - 1;
//alert(rows);
var page = '<html><div align="left"><a href="javascript:void(0);" id="back" style="font-size:16px;color:blue">Click to Return</a>';
page = page + '<a id="download" style="font-size:16px;margin-left:20px">Click to Download</a>';
page = page + '<a href="/" id="trick" style="display: none">Click to back</a>';
page = page + '<p>Total answers: ';
page = page + parts[1];
page = page + '</p>';
if(parts[0] == "1")
{
page = page + '<p>Number of answers is too large, show only 100 of them, click to download the full result!</p>';
}
page = page + '</div><table border="1" style = "font-size:medium">';
page = page + "<tr>";
for(var ii = 1; ii <= rows; ii++)
{
page = page + "<th>" + "?" + valNum[ii] + "</th>";
}
page = page + "</tr>";
var i = 1;
var j = 0;
for (i = 1; i <= lines; i++)
{
page = page + "<tr>";
//alert(items[i]);
var tmpItem = items[i].replace(/"([^"]*)"/g, "<$1>");
//alert(tmpItem);
var item = tmpItem.split(">");
//alert(item.length);
for(j = 0; j < rows; j++)
{
//alert(item[j]);
page = page + '<td>' + item[j].replace("<", "") + '</td>';
}
page = page + "</tr>";
}
page = page + '</table></div></html>';
$("#main_body").empty();
$("#main_body").append(page);
var tmp1 = "?operation=download&filepath=" + fileName;
var request1 = escape(tmp1);
var element1 = document.getElementById("download");
element1.setAttribute("href", request1);
element1.setAttribute("download", "sparql.txt");
var tmp2 = "?operation=delete&filepath=" + fileName;
var request2 = escape(tmp2);
var element2 = document.getElementById("back");
//!Notice: element2 is a "<a>" tag, and it has two actions, href and onclick, be careful with the executing order of these two actions.
//in this case, we use a call-back function to prevent strange things. we return to the origin web page after the request to delete file returns successfully.
element2.onclick = function(){
$.get(request2, function(data, status){
//alert("delete return");
var element3 = document.getElementById("trick");
if(/msie/i.test(navigator.userAgent))
{
element3.fireEvent("onclick");
}
else
{
var e1 = document.createEvent("MouseEvents");
e1.initEvent("click", true, true);
element3.dispatchEvent(e1);
}
});
}
}
});
}
}
function handleQueryExample()
{
var example = document.getElementById("example").value;
if(example === "q1")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t?x <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> ?y .\n" +
"}\n";
}
if(example === "q2")
{
document.getElementById("element_3").value = "select ?x\n" +
"where\n" +
"{ \n" +
"\t?s <http://wasp.cs.vu.nl/apdg#hasArchetype> ?y .\n" +
"\t?y <http://wasp.cs.vu.nl/apdg#Concept> ?x .\n" +
"} \n";
}
if(example === "q3")
{
document.getElementById("element_3").value = "select ?x \n" +
"where\n" +
"{ \n" +
"\t<http://wasp.cs.vu.nl/apdg#VMRBC_ZSH2013A1001999_A1> ?p ?z .\n" +
"\t?z <http://wasp.cs.vu.nl/apdg#hasArchetype> ?y .\n" +
"\t?y <http://wasp.cs.vu.nl/apdg#hasSlot> ?x .\n" +
"} \n";
}
//if (example === "q4")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:GraduateStudent>. \n" +
//"\t?y <rdf:type> <ub:University>. \n" +
//"\t?z <rdf:type> <ub:Department>. \n" +
//"\t?x <ub:memberOf> ?z. \n" +
//"\t?z <ub:subOrganizationOf> ?y. \n" +
//"\t?x <ub:undergraduateDegreeFrom> ?y. \n" +
//"} \n";
//}
//if (example === "q5")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:Course>. \n" +
//"\t?x <ub:name> ?y. \n" +
//"} \n";
//}
//if (example === "q6")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"\t?y <ub:name> <Course1>. \n" +
//"\t?x <ub:takesCourse> ?y. \n" +
//"\t?z <ub:teacherOf> ?y. \n" +
//"\t?z <ub:name> <FullProfessor1>. \n" +
//"\t?z <ub:worksFor> ?w. \n" +
//"\t?w <ub:name> <Department0>. \n" +
//"} \n";
//}
//if (example === "q7")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"}\n";
//}
//if (example === "q8")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 ?v4 ?v5 ?v6 WHERE \n" +
//"{\n" +
//"\t?v0 <http://www.w3.org/2004/02/skos/core#broader> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/child> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/spouse> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/profession> ?v4 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/party> ?v5 .\n" +
//"\t?v0 <http://dbpedia.prg/ontology/militaryRank> ?v6 .\n" +
//"}\n";
//}
//if (example === "q9")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?V0 <http://dbpedia.org/ontology/location> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/typeOfElectrification> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Koltsevaya_Line> .\n" +
//"\t?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v3 .\n" +
//"}\n";
//}
//if (example === "q10")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?v0 <http://dbpedia.org/property/hasPhotoCollection> ?v1 .\n" +
//"\t?v0 <xmlns.com/foaf/0.1/depiction> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/abstract> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Combination> .\n" +
//"}\n";
//}
}

82
Server/web/openkg/clinga/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>OpenKG Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>OpenKG Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>OpenKG Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="clinga" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<!--<option value="q4">q4</option>-->
<!--<option value="q5">q5</option>-->
<!--<option value="q6">q6</option>-->
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

241
Server/web/openkg/clinga/index.js Normal file
View File

@ -0,0 +1,241 @@
function query(dp) {
var encodeVal = escape(dp);
//alert(encodeVal);
var format = document.getElementById("element_5").value;
if(format == "1")
format = "html";
else if(format == "2")
format = "txt";
else if(format == "3")
format = "csv";
else if(format == "4")
format = "json";
var argu = "?operation=query&format=" + format + "&sparql=" + dp;
var encodeArgu = escape(argu);
if(format != "html")
{
/*
$.get(encodeArgu, function(data, status){
if(status == "success")
{
alert("success");
}
});
*/
var element = document.getElementById("hideLink");
element.setAttribute("href", encodeArgu);
/*
if(format == "txt")
element.setAttribute("download", "sparql.txt");
else if(format == "csv")
element.setAttribute("download", "sparql.csv");
else if(format == "json")
element.setAttribute("download", "sparql.json");
*/
if(/msie/i.test(navigator.userAgent))
{
element.fireEvent("onclick");
}
else
{
var e = document.createEvent("MouseEvents");
e.initEvent("click", true, true);
element.dispatchEvent(e);
}
}
else
{
$.get(encodeArgu, function(data, status){
if(status=="success"){
//toTxt();
//alert(data);
var parts = data.split("+");
var fileName = parts[2];
var lines = Number(parts[1]);
//alert(lines);
if(lines > 100)
lines = 100;
//alert(lines);
var items = parts[3].split("\n");
//alert(items[0]);
var valNum = items[0].split("?");
var rows = valNum.length - 1;
//alert(rows);
var page = '<html><div align="left"><a href="javascript:void(0);" id="back" style="font-size:16px;color:blue">Click to Return</a>';
page = page + '<a id="download" style="font-size:16px;margin-left:20px">Click to Download</a>';
page = page + '<a href="/" id="trick" style="display: none">Click to back</a>';
page = page + '<p>Total answers: ';
page = page + parts[1];
page = page + '</p>';
if(parts[0] == "1")
{
page = page + '<p>Number of answers is too large, show only 100 of them, click to download the full result!</p>';
}
page = page + '</div><table border="1" style = "font-size:medium">';
page = page + "<tr>";
for(var ii = 1; ii <= rows; ii++)
{
page = page + "<th>" + "?" + valNum[ii] + "</th>";
}
page = page + "</tr>";
var i = 1;
var j = 0;
for (i = 1; i <= lines; i++)
{
page = page + "<tr>";
//alert(items[i]);
var tmpItem = items[i].replace(/"([^"]*)"/g, "<$1>");
//alert(tmpItem);
var item = tmpItem.split(">");
//alert(item.length);
for(j = 0; j < rows; j++)
{
//alert(item[j]);
page = page + '<td>' + item[j].replace("<", "") + '</td>';
}
page = page + "</tr>";
}
page = page + '</table></div></html>';
$("#main_body").empty();
$("#main_body").append(page);
var tmp1 = "?operation=download&filepath=" + fileName;
var request1 = escape(tmp1);
var element1 = document.getElementById("download");
element1.setAttribute("href", request1);
element1.setAttribute("download", "sparql.txt");
var tmp2 = "?operation=delete&filepath=" + fileName;
var request2 = escape(tmp2);
var element2 = document.getElementById("back");
//!Notice: element2 is a "<a>" tag, and it has two actions, href and onclick, be careful with the executing order of these two actions.
//in this case, we use a call-back function to prevent strange things. we return to the origin web page after the request to delete file returns successfully.
element2.onclick = function(){
$.get(request2, function(data, status){
//alert("delete return");
var element3 = document.getElementById("trick");
if(/msie/i.test(navigator.userAgent))
{
element3.fireEvent("onclick");
}
else
{
var e1 = document.createEvent("MouseEvents");
e1.initEvent("click", true, true);
element3.dispatchEvent(e1);
}
});
}
}
});
}
}
function handleQueryExample()
{
var example = document.getElementById("example").value;
if(example === "q1")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t<http://ws.nju.edu.cn/clinga/A2822147> ?x ?y .\n" +
"}\n";
}
if(example === "q2")
{
document.getElementById("element_3").value = "select ?x ?y\n" +
"where\n" +
"{ \n" +
"\t<http://ws.nju.edu.cn/clinga/A2822147> ?p ?z .\n" +
"\t?z ?x ?y.\n" +
"} \n";
}
if(example === "q3")
{
document.getElementById("element_3").value = "select ?x ?y \n" +
"where\n" +
"{ \n" +
"\t<http://ws.nju.edu.cn/clinga/A2822147> ?p ?z .\n" +
"\t?z <http://ws.nju.edu.cn/clinga/infoboxKey> ?x .\n" +
"\t?z <http://ws.nju.edu.cn/clinga/infoboxValue> ?y .\n" +
"} \n";
}
//if (example === "q4")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:GraduateStudent>. \n" +
//"\t?y <rdf:type> <ub:University>. \n" +
//"\t?z <rdf:type> <ub:Department>. \n" +
//"\t?x <ub:memberOf> ?z. \n" +
//"\t?z <ub:subOrganizationOf> ?y. \n" +
//"\t?x <ub:undergraduateDegreeFrom> ?y. \n" +
//"} \n";
//}
//if (example === "q5")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:Course>. \n" +
//"\t?x <ub:name> ?y. \n" +
//"} \n";
//}
//if (example === "q6")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"\t?y <ub:name> <Course1>. \n" +
//"\t?x <ub:takesCourse> ?y. \n" +
//"\t?z <ub:teacherOf> ?y. \n" +
//"\t?z <ub:name> <FullProfessor1>. \n" +
//"\t?z <ub:worksFor> ?w. \n" +
//"\t?w <ub:name> <Department0>. \n" +
//"} \n";
//}
//if (example === "q7")
//{
//document.getElementById("element_3").value = "select distinct ?x where \n" +
//"{ \n" +
//"\t?x <rdf:type> <ub:UndergraduateStudent>. \n" +
//"}\n";
//}
//if (example === "q8")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 ?v4 ?v5 ?v6 WHERE \n" +
//"{\n" +
//"\t?v0 <http://www.w3.org/2004/02/skos/core#broader> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/child> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/spouse> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/profession> ?v4 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/party> ?v5 .\n" +
//"\t?v0 <http://dbpedia.prg/ontology/militaryRank> ?v6 .\n" +
//"}\n";
//}
//if (example === "q9")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?V0 <http://dbpedia.org/ontology/location> ?v1 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/typeOfElectrification> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Koltsevaya_Line> .\n" +
//"\t?v0 <http://dbpedia.org/property/wikiPageUsesTemplate> ?v3 .\n" +
//"}\n";
//}
//if (example === "q10")
//{
//document.getElementById("element_3").value = "SELECT ?v0 ?v1 ?v2 ?v3 WHERE\n" +
//"{\n" +
//"\t?v0 <http://dbpedia.org/property/hasPhotoCollection> ?v1 .\n" +
//"\t?v0 <xmlns.com/foaf/0.1/depiction> ?v2 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/abstract> ?v3 .\n" +
//"\t?v0 <http://dbpedia.org/ontology/wikiPageWikiLink> <http://dbpedia.org/resource/Combination> .\n" +
//"}\n";
//}
}

82
Server/web/openkg/emergency/index.html Normal file
View File

@ -0,0 +1,82 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>OpenKG Query Endpoint</title>
<link rel="stylesheet" type="text/css" href="view.css" media="all" />
<!--<script type="text/javascript" src="view.js"></script> -->
<script type="text/javascript" src="index.js"></script>
</head>
<body id="main_body" onload="handleQueryExample()">
<div id="form_container">
<h1><a>OpenKG Query Endpoint</a></h1>
<form id="myForm" method="post" action="PHPAPIExample.php">
<div class="form_description">
<h2>OpenKG Query Endpoint</h2>
<p></p>
</div>
<ul >
<li id="li_1" >
<label class="description" for="element_1">Database Name </label>
<div>
<input id="element_1" name="databasename" class="element text medium" type="text" maxlength="255" value="emergency" readonly>
</input>
</div>
</li>
<li id="li_2" >
<label class="description" for="element_2">Query Examples</label>
<div>
<select id="example" class="element select medium" onchange="handleQueryExample()">
<option value="q1" selected="selected">q1</option>
<option value="q2">q2</option>
<option value="q3">q3</option>
<!--<option value="q4">q4</option>-->
<!--<option value="q5">q5</option>-->
<!--<option value="q6">q6</option>-->
<!--<option value="q7">q7</option>-->
<!--<option value="q8">q8</option>-->
<!--<option value="q9">q9</option>-->
<!--<option value="q10">q10</option>-->
</select>
</div>
</li>
<li id="li_3" >
<label class="description" for="element_3">Query Text </label>
<div>
<textarea id="element_3" name="query_file" class="element textarea large"></textarea>
</div>
</li>
<li id="li_5" >
<label class="description" for="element_5">Results Format </label>
<div>
<select class="element select medium" id="element_5" name="format">
<option value="1" selected="ture">HTML</option>
<option value="2" >Text</option>
<option value="3" >CSV</option>
<option value="4" >JSON</option>
</select>
</div>
</li>
</ul>
<input type="hidden" name="queryAns1" id="queryAns" value="defaultAns" />
<a id="hideLink" style="display: none">link</a>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">Run Query</button>
</form>
<div id="footer">
</div>
</div>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<!--<script src="index.js"></script>-->
</body>
</html>

Some files were not shown because too many files have changed in this diff Show More