opensci
/
gStore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: use multithreading to load; 

fix type bugs in string index;

by zengli
This commit is contained in:
bookug 2017-06-06 18:56:46 +08:00
parent 886ca8512d
commit fa147526f1
7 changed files with 252 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
Database/Database.cpp
View File

@ -572,14 +572,30 @@ Database::load()
} }
//TODO: acquire this arg from memory manager //TODO: acquire this arg from memory manager
//BETTER: get return value from subthread(using ref or file as hub)
unsigned vstree_cache = LRUCache::DEFAULT_CAPACITY; unsigned vstree_cache = LRUCache::DEFAULT_CAPACITY;
bool flag = (this->vstree)->loadTree(vstree_cache); thread vstree_thread(&Database::load_vstree, this, vstree_cache);
if (!flag) bool flag;
{ //flag = (this->vstree)->loadTree(vstree_cache);
cout << "load tree error. @Database::load()" << endl; //if (!flag)
return false; //{
} //cout << "load tree error. @Database::load()" << endl;
//return false;
//}
//(this->kvstore)->open();
int kv_mode = KVstore::READ_WRITE_MODE;
thread entity2id_thread(&Database::load_entity2id, this, kv_mode);
thread id2entity_thread(&Database::load_id2entity, this, kv_mode);
thread literal2id_thread(&Database::load_literal2id, this, kv_mode);
thread id2literal_thread(&Database::load_id2literal, this, kv_mode);
thread predicate2id_thread(&Database::load_predicate2id, this, kv_mode);
thread id2predicate_thread(&Database::load_id2predicate, this, kv_mode);
thread sub2values_thread(&Database::load_sub2values, this, kv_mode);
thread obj2values_thread(&Database::load_obj2values, this, kv_mode);
thread pre2values_thread(&Database::load_pre2values, this, kv_mode);
//this is very fast
flag = this->loadDBInfoFile(); flag = this->loadDBInfoFile();
if (!flag) if (!flag)
{ {
@ -587,18 +603,30 @@ Database::load()
return false; return false;
} }
(this->kvstore)->open(); //NOTICE: we should also run some heavy work in the main thread
this->stringindex->load(); this->stringindex->load();
this->readIDinfo(); this->readIDinfo();
pre2values_thread.join();
this->setPreMap(); this->setPreMap();
id2entity_thread.join();
id2literal_thread.join();
//generate the string buffer for entity and literal, no need for predicate //generate the string buffer for entity and literal, no need for predicate
//NOTICE:the total string size should not exceed 10G, assume that most strings length < 500 //NOTICE:the total string size should not exceed 10G, assume that most strings length < 500
//too many empty between entity and literal, so divide them //too many empty between entity and literal, so divide them
this->setStringBuffer(); this->setStringBuffer();
id2predicate_thread.join();
entity2id_thread.join();
literal2id_thread.join();
predicate2id_thread.join();
sub2values_thread.join();
obj2values_thread.join();
//wait for vstree thread
vstree_thread.join();
//warm up always as finishing build(), to utilize the system buffer //warm up always as finishing build(), to utilize the system buffer
//this->warmUp(); //this->warmUp();
//DEBUG:the warmUp() calls query(), which will also output results, this is not we want //DEBUG:the warmUp() calls query(), which will also output results, this is not we want
@ -606,8 +634,109 @@ Database::load()
this->if_loaded = true; this->if_loaded = true;
cout << "finish load" << endl; cout << "finish load" << endl;
//TODO: for only-read application(like endpoint), 3 id2values trees can be closed now
//and we should load all trees on only READ mode
//HELP: just for checking infos(like kvstore)
check();
return true; return true;
} }
void
Database::load_entity2id(int _mode)
{
this->kvstore->open_entity2id(_mode);
}
void
Database::load_id2entity(int _mode)
{
this->kvstore->open_id2entity(_mode);
}
void
Database::load_literal2id(int _mode)
{
this->kvstore->open_literal2id(_mode);
}
void
Database::load_id2literal(int _mode)
{
this->kvstore->open_id2literal(_mode);
}
void
Database::load_predicate2id(int _mode)
{
this->kvstore->open_predicate2id(_mode);
}
void
Database::load_id2predicate(int _mode)
{
this->kvstore->open_id2predicate(_mode);
}
void
Database::load_sub2values(int _mode)
{
this->kvstore->open_subID2values(_mode);
}
void
Database::load_obj2values(int _mode)
{
this->kvstore->open_objID2values(_mode);
}
void
Database::load_pre2values(int _mode)
{
this->kvstore->open_preID2values(_mode);
}
void
Database::load_vstree(unsigned _vstree_size)
{
(this->vstree)->loadTree(_vstree_size);
cout<<"vstree loaded"<<endl;
}
void
Database::check()
{
//unsigned pid = this->kvstore->getIDByPredicate("<http://www.w3.org/2000/01/rdf-schema#label>");
//cout<<"check: pre "<<pid<<endl;
//unsigned sid = this->kvstore->getIDByEntity("<http://rdf.freebase.com/ns/american_football.football_player>");
//unsigned oid = this->kvstore->getIDByLiteral("\"Игроки в американский футбол\"@ru");
//unsigned* list = NULL; unsigned len = 0;
//this->kvstore->getobjIDlistBysubIDpreID(sid, pid, list, len);
//FILE* fp = fopen("kv.txt", "w");
//for(unsigned i = 0; i < len; ++i)
//{
//fprintf(fp, "%u\n", list[i]);
//string ts;
//if(Util::is_literal_ele(list[i]))
//ts = this->kvstore->getLiteralByID(list[i]);
//else
//ts = this->kvstore->getEntityByID(list[i]);
//if(ts == "")
//{
//fprintf(fp, "Error in id2string\n");
//}
//else
//{
//fprintf(fp, "%s\n", ts.c_str());
//}
//}
//string tstr;
//this->stringindex->randomAccess(2164939819, &tstr, true);
//cout<<"check: 2164939819 "<<tstr<<endl;
//this->stringindex->randomAccess(2164939818, &tstr, true);
//cout<<"check: 2164939818 "<<tstr<<endl;
//fclose(fp);
}
//NOTICE: we ensure that if the unload() exists normally, then all updates have already been written to disk //NOTICE: we ensure that if the unload() exists normally, then all updates have already been written to disk
//So when accidents happens, we only have to restore the databases that are in load status(inlucidng that unload //So when accidents happens, we only have to restore the databases that are in load status(inlucidng that unload

13
Database/Database.h
View File

@ -131,6 +131,19 @@ private:
//BETTER:add a predicate buffer for ?p query //BETTER:add a predicate buffer for ?p query
//However, I think this is not necessary because ?p is rare and the p2xx tree is small enough //However, I think this is not necessary because ?p is rare and the p2xx tree is small enough
void check();
//used for multiple threads
void load_vstree(unsigned _vstree_size);
void load_entity2id(int _mode);
void load_id2entity(int _mode);
void load_literal2id(int _mode);
void load_id2literal(int _mode);
void load_predicate2id(int _mode);
void load_id2predicate(int _mode);
void load_sub2values(int _mode);
void load_obj2values(int _mode);
void load_pre2values(int _mode);
//triple num per group for insert/delete //triple num per group for insert/delete
//can not be too high, otherwise the heap will over //can not be too high, otherwise the heap will over
static const int GROUP_SIZE = 1000; static const int GROUP_SIZE = 1000;

90
StringIndex/StringIndex.cpp
View File

@ -10,7 +10,7 @@
using namespace std; using namespace std;
void StringIndexFile::setNum(int _num) void StringIndexFile::setNum(unsigned _num)
{ {
this->num = _num; this->num = _num;
} }
@ -33,7 +33,7 @@ void StringIndexFile::save(KVstore &kv_store)
fwrite(&this->num, sizeof(int), 1, this->index_file); fwrite(&this->num, sizeof(int), 1, this->index_file);
long offset = 0; long offset = 0;
for (int i = 0; i < this->num; i++) for (unsigned i = 0; i < this->num; i++)
{ {
string str; string str;
if (this->type == Entity) if (this->type == Entity)
@ -43,7 +43,7 @@ void StringIndexFile::save(KVstore &kv_store)
if (this->type == Predicate) if (this->type == Predicate)
str = kv_store.getPredicateByID(i); str = kv_store.getPredicateByID(i);
int length = str.length(); unsigned length = str.length();
fwrite(&offset, sizeof(long), 1, this->index_file); fwrite(&offset, sizeof(long), 1, this->index_file);
fwrite(&length, sizeof(int), 1, this->index_file); fwrite(&length, sizeof(int), 1, this->index_file);
offset += length; offset += length;
@ -74,7 +74,7 @@ void StringIndexFile::load()
fread(&this->num, sizeof(int), 1, this->index_file); fread(&this->num, sizeof(int), 1, this->index_file);
this->index_table.resize(this->num); this->index_table.resize(this->num);
for (int i = 0; i < this->num; i++) for (unsigned i = 0; i < this->num; i++)
{ {
fread(&this->index_table[i].offset, sizeof(long), 1, this->index_file); fread(&this->index_table[i].offset, sizeof(long), 1, this->index_file);
fread(&this->index_table[i].length, sizeof(int), 1, this->index_file); fread(&this->index_table[i].length, sizeof(int), 1, this->index_file);
@ -82,13 +82,15 @@ void StringIndexFile::load()
} }
} }
bool StringIndexFile::randomAccess(int id, string *str) bool StringIndexFile::randomAccess(unsigned id, string *str)
{ {
if (id < 0 || id >= this->num) //DEBUG: int or unsigned here???
//if (id < 0 || id >= this->num)
if (id >= this->num)
return false; return false;
long offset = this->index_table[id].offset; long offset = this->index_table[id].offset;
int length = this->index_table[id].length; unsigned length = this->index_table[id].length;
allocBuffer(length); allocBuffer(length);
@ -97,11 +99,16 @@ bool StringIndexFile::randomAccess(int id, string *str)
this->buffer[length] = '\0'; this->buffer[length] = '\0';
*str = string(this->buffer); *str = string(this->buffer);
//cout<<"check: read from string index - "<<id<<" "<<*str<<endl;
if(*str == "")
{
cout<<"ERROR in StringIndex - "<<id<<endl;
}
return true; return true;
} }
void StringIndexFile::addRequest(int id, std::string *str) void StringIndexFile::addRequest(unsigned id, std::string *str)
{ {
this->request.push_back(AccessRequest(id, this->index_table[id].offset, this->index_table[id].length, str)); this->request.push_back(AccessRequest(id, this->index_table[id].offset, this->index_table[id].length, str));
} }
@ -109,7 +116,7 @@ void StringIndexFile::addRequest(int id, std::string *str)
void StringIndexFile::trySequenceAccess() void StringIndexFile::trySequenceAccess()
{ {
long max_end = 0; long max_end = 0;
for (int i = 0; i < (int)this->request.size(); i++) for (unsigned i = 0; i < this->request.size(); i++)
max_end = max(max_end, this->request[i].offset + long(this->request[i].length)); max_end = max(max_end, this->request[i].offset + long(this->request[i].length));
if (this->type == Entity) if (this->type == Entity)
@ -124,7 +131,7 @@ void StringIndexFile::trySequenceAccess()
sort(this->request.begin(), this->request.end()); sort(this->request.begin(), this->request.end());
int pos = 0; unsigned pos = 0;
fseek(this->value_file, 0, SEEK_SET); fseek(this->value_file, 0, SEEK_SET);
char *block = new char[MAX_BLOCK_SIZE]; char *block = new char[MAX_BLOCK_SIZE];
long current_block_begin = 0; long current_block_begin = 0;
@ -145,6 +152,10 @@ void StringIndexFile::trySequenceAccess()
memcpy(this->buffer, &block[offset - current_block_begin], length); memcpy(this->buffer, &block[offset - current_block_begin], length);
this->buffer[length] = '\0'; this->buffer[length] = '\0';
*this->request[pos].str = string(this->buffer); *this->request[pos].str = string(this->buffer);
if(string(this->buffer) == "")
{
cout<<"Error in StringIndex"<<endl;
}
pos++; pos++;
} }
else if (current_block_begin <= offset) else if (current_block_begin <= offset)
@ -154,6 +165,10 @@ void StringIndexFile::trySequenceAccess()
memcpy(this->buffer, &block[offset - current_block_begin], length); memcpy(this->buffer, &block[offset - current_block_begin], length);
this->buffer[length] = '\0'; this->buffer[length] = '\0';
*this->request[pos].str = string(this->buffer); *this->request[pos].str = string(this->buffer);
if(string(this->buffer) == "")
{
cout<<"Error in StringIndex"<<endl;
}
break; break;
} }
else if (offset + length <= current_block_end) else if (offset + length <= current_block_end)
@ -167,6 +182,10 @@ void StringIndexFile::trySequenceAccess()
while (pos < (int)this->request.size() && this->request[pos - 1].offset == this->request[pos].offset) while (pos < (int)this->request.size() && this->request[pos - 1].offset == this->request[pos].offset)
{ {
*this->request[pos].str = *this->request[pos - 1].str; *this->request[pos].str = *this->request[pos - 1].str;
if(*this->request[pos].str == "")
{
cout<<"Error in StringIndex"<<endl;
}
pos++; pos++;
} }
} }
@ -177,6 +196,10 @@ void StringIndexFile::trySequenceAccess()
memcpy(this->buffer, block, length); memcpy(this->buffer, block, length);
this->buffer[length] = '\0'; this->buffer[length] = '\0';
*this->request[pos].str += string(this->buffer); *this->request[pos].str += string(this->buffer);
if(*this->request[pos].str == "")
{
cout<<"Error in StringIndex"<<endl;
}
break; break;
} }
} }
@ -189,16 +212,23 @@ void StringIndexFile::trySequenceAccess()
{ {
cout << "random access." << endl; cout << "random access." << endl;
for (int i = 0; i < (int)this->request.size(); i++) for (unsigned i = 0; i < (int)this->request.size(); i++)
this->randomAccess(this->request[i].id, this->request[i].str); this->randomAccess(this->request[i].id, this->request[i].str);
} }
this->request.clear(); this->request.clear();
} }
void StringIndexFile::change(int id, KVstore &kv_store) void StringIndexFile::change(unsigned id, KVstore &kv_store)
{ {
if (id < 0) return; if(this->type == Predicate)
{
if (id < 0) return;
}
else
{
if (id == INVALID) return;
}
if (this->num <= id) if (this->num <= id)
{ {
@ -237,9 +267,16 @@ void StringIndexFile::change(int id, KVstore &kv_store)
fwrite(str.c_str(), sizeof(char), this->index_table[id].length, this->value_file); fwrite(str.c_str(), sizeof(char), this->index_table[id].length, this->value_file);
} }
void StringIndexFile::disable(int id) void StringIndexFile::disable(unsigned id)
{ {
if (id < 0 || id >= this->num) return ; if(this->type == Predicate)
{
if (id < 0 || id >= this->num) return ;
}
else
{
if (id == INVALID) return;
}
this->index_table[id] = IndexInfo(); this->index_table[id] = IndexInfo();
@ -249,7 +286,7 @@ void StringIndexFile::disable(int id)
} }
//---------------------------------------------------------------------------------------------------------------------------------------------------- //----------------------------------------------------------------------------------------------------------------------------------------------------
void StringIndex::setNum(StringIndexFile::StringIndexFileType _type, int _num) void StringIndex::setNum(StringIndexFile::StringIndexFileType _type, unsigned _num)
{ {
if (_type == StringIndexFile::Entity) if (_type == StringIndexFile::Entity)
this->entity.setNum(_num); this->entity.setNum(_num);
@ -274,7 +311,7 @@ void StringIndex::load()
} }
bool bool
StringIndex::searchBuffer(int _id, string* _str) StringIndex::searchBuffer(unsigned _id, string* _str)
{ {
if(_id < Util::LITERAL_FIRST_ID) //entity if(_id < Util::LITERAL_FIRST_ID) //entity
{ {
@ -297,7 +334,7 @@ StringIndex::searchBuffer(int _id, string* _str)
} }
} }
bool StringIndex::randomAccess(int id, string *str, bool is_entity_or_literal) bool StringIndex::randomAccess(unsigned id, string *str, bool is_entity_or_literal)
{ {
if(id < 0) return false; if(id < 0) return false;
@ -307,11 +344,18 @@ bool StringIndex::randomAccess(int id, string *str, bool is_entity_or_literal)
{ {
return true; return true;
} }
else
{
//cout<<"check: not found in string buffer - "<<id<<endl;
}
if (id < Util::LITERAL_FIRST_ID) if (id < Util::LITERAL_FIRST_ID)
return this->entity.randomAccess(id, str); return this->entity.randomAccess(id, str);
else else
{
//cout<<"check: to search literal "<<id-Util::LITERAL_FIRST_ID<<endl;
return this->literal.randomAccess(id - Util::LITERAL_FIRST_ID, str); return this->literal.randomAccess(id - Util::LITERAL_FIRST_ID, str);
}
} }
else else
{ {
@ -319,7 +363,7 @@ bool StringIndex::randomAccess(int id, string *str, bool is_entity_or_literal)
} }
} }
void StringIndex::addRequest(int id, std::string *str, bool is_entity_or_literal) void StringIndex::addRequest(unsigned id, std::string *str, bool is_entity_or_literal)
{ {
if (is_entity_or_literal) if (is_entity_or_literal)
{ {
@ -349,7 +393,7 @@ void StringIndex::change(std::vector<unsigned> &ids, KVstore &kv_store, bool is_
{ {
if (is_entity_or_literal) if (is_entity_or_literal)
{ {
for (int i = 0; i < (int)ids.size(); i++) for (unsigned i = 0; i < ids.size(); i++)
{ {
if (ids[i] < Util::LITERAL_FIRST_ID) if (ids[i] < Util::LITERAL_FIRST_ID)
this->entity.change(ids[i], kv_store); this->entity.change(ids[i], kv_store);
@ -359,7 +403,7 @@ void StringIndex::change(std::vector<unsigned> &ids, KVstore &kv_store, bool is_
} }
else else
{ {
for (int i = 0; i < (int)ids.size(); i++) for (unsigned i = 0; i < ids.size(); i++)
this->predicate.change(ids[i], kv_store); this->predicate.change(ids[i], kv_store);
} }
} }
@ -368,7 +412,7 @@ void StringIndex::disable(std::vector<unsigned> &ids, bool is_entity_or_literal)
{ {
if (is_entity_or_literal) if (is_entity_or_literal)
{ {
for (int i = 0; i < (int)ids.size(); i++) for (unsigned i = 0; i < ids.size(); i++)
{ {
if (ids[i] < Util::LITERAL_FIRST_ID) if (ids[i] < Util::LITERAL_FIRST_ID)
this->entity.disable(ids[i]); this->entity.disable(ids[i]);
@ -378,7 +422,7 @@ void StringIndex::disable(std::vector<unsigned> &ids, bool is_entity_or_literal)
} }
else else
{ {
for (int i = 0; i < (int)ids.size(); i++) for (unsigned i = 0; i < ids.size(); i++)
this->predicate.disable(ids[i]); this->predicate.disable(ids[i]);
} }
} }

38
StringIndex/StringIndex.h
View File

@ -22,30 +22,30 @@ class StringIndexFile
private: private:
StringIndexFileType type; StringIndexFileType type;
std::string loc; std::string loc;
int num; unsigned num;
long empty_offset; long empty_offset;
FILE *index_file, *value_file; FILE *index_file, *value_file;
class IndexInfo class IndexInfo
{ {
public: public:
IndexInfo(long _offset = 0, int _length = 0):offset(_offset), length(_length){} IndexInfo(long _offset = 0, unsigned _length = 0):offset(_offset), length(_length){}
long offset; long offset;
int length; unsigned length;
}; };
std::vector<IndexInfo> index_table; std::vector<IndexInfo> index_table;
int buffer_size; unsigned buffer_size;
char *buffer; char *buffer;
class AccessRequest class AccessRequest
{ {
public: public:
int id; unsigned id;
long offset; long offset;
int length; unsigned length;
std::string *str; std::string *str;
AccessRequest(int _id, long _offset, int _length, std::string *_str): AccessRequest(unsigned _id, long _offset, unsigned _length, std::string *_str):
id(_id), offset(_offset), length(_length), str(_str){}; id(_id), offset(_offset), length(_length), str(_str){};
inline bool operator < (const AccessRequest &x) const inline bool operator < (const AccessRequest &x) const
{ {
@ -55,7 +55,7 @@ class StringIndexFile
std::vector<AccessRequest> request; std::vector<AccessRequest> request;
public: public:
StringIndexFile(StringIndexFileType _type, std::string _dir, int _num):type(_type), num(_num), empty_offset(0), index_file(NULL), value_file(NULL), buffer_size(0), buffer(NULL) StringIndexFile(StringIndexFileType _type, std::string _dir, unsigned _num):type(_type), num(_num), empty_offset(0), index_file(NULL), value_file(NULL), buffer_size(0), buffer(NULL)
{ {
if (this->type == Entity) if (this->type == Entity)
this->loc = _dir + "/entity_"; this->loc = _dir + "/entity_";
@ -73,12 +73,12 @@ class StringIndexFile
if (this->buffer != NULL) if (this->buffer != NULL)
delete[] this->buffer; delete[] this->buffer;
} }
void setNum(int _num); void setNum(unsigned _num);
void save(KVstore &kv_store); void save(KVstore &kv_store);
void load(); void load();
inline void allocBuffer(int length) inline void allocBuffer(unsigned length)
{ {
if (this->buffer_size <= length) if (this->buffer_size <= length)
{ {
@ -88,12 +88,12 @@ class StringIndexFile
} }
} }
bool randomAccess(int id, std::string *str); bool randomAccess(unsigned id, std::string *str);
void addRequest(int id, std::string *str); void addRequest(unsigned id, std::string *str);
void trySequenceAccess(); void trySequenceAccess();
void change(int id, KVstore &kv_store); void change(unsigned id, KVstore &kv_store);
void disable(int id); void disable(unsigned id);
}; };
class StringIndex class StringIndex
@ -105,7 +105,7 @@ class StringIndex
Buffer* literal_buffer; Buffer* literal_buffer;
unsigned literal_buffer_size; unsigned literal_buffer_size;
public: public:
StringIndex(std::string _dir, int _entity_num = 0, int _literal_num = 0, int _predicate_num = 0) : StringIndex(std::string _dir, unsigned _entity_num = 0, unsigned _literal_num = 0, unsigned _predicate_num = 0) :
entity(StringIndexFile::Entity, _dir, _entity_num), literal(StringIndexFile::Literal, _dir, _literal_num), predicate(StringIndexFile::Predicate, _dir, _predicate_num){} entity(StringIndexFile::Entity, _dir, _entity_num), literal(StringIndexFile::Literal, _dir, _literal_num), predicate(StringIndexFile::Predicate, _dir, _predicate_num){}
void setBuffer(Buffer* _ebuf, Buffer* _lbuf) void setBuffer(Buffer* _ebuf, Buffer* _lbuf)
@ -115,15 +115,15 @@ class StringIndex
this->literal_buffer = _lbuf; this->literal_buffer = _lbuf;
this->literal_buffer_size = _lbuf->size; this->literal_buffer_size = _lbuf->size;
} }
bool searchBuffer(int _id, std::string* _str); bool searchBuffer(unsigned _id, std::string* _str);
void setNum(StringIndexFile::StringIndexFileType _type, int _num); void setNum(StringIndexFile::StringIndexFileType _type, unsigned _num);
void save(KVstore &kv_store); void save(KVstore &kv_store);
void load(); void load();
bool randomAccess(int id, std::string *str, bool is_entity_or_literal = true); bool randomAccess(unsigned id, std::string *str, bool is_entity_or_literal = true);
void addRequest(int id, std::string *str, bool is_entity_or_literal = true); void addRequest(unsigned id, std::string *str, bool is_entity_or_literal = true);
void trySequenceAccess(); void trySequenceAccess();
void change(std::vector<unsigned> &ids, KVstore &kv_store, bool is_entity_or_literal = true); void change(std::vector<unsigned> &ids, KVstore &kv_store, bool is_entity_or_literal = true);

11
Util/Util.h
View File

@ -56,6 +56,7 @@ in the sparql query can point to the same node in data graph)
#include <set> #include <set>
#include <stack> #include <stack>
#include <queue> #include <queue>
#include <deque>
#include <vector> #include <vector>
#include <list> #include <list>
#include <iterator> #include <iterator>
@ -64,6 +65,13 @@ in the sparql query can point to the same node in data graph)
#include <utility> #include <utility>
//NOTICE:below are libraries need to link //NOTICE:below are libraries need to link
#include <thread> //only for c++11 or greater versions
#include <atomic>
#include <mutex>
#include <condition_variable>
#include <future>
#include <memory>
#include <stdexcept>
#include <pthread.h> #include <pthread.h>
#include <math.h> #include <math.h>
#include <readline/readline.h> #include <readline/readline.h>
@ -210,7 +218,8 @@ public:
//In order to differentiate the sub-part and literal-part of object //In order to differentiate the sub-part and literal-part of object
//let subid begin with 0, while literalid begins with LITERAL_FIRST_ID //let subid begin with 0, while literalid begins with LITERAL_FIRST_ID
//used in Database and Join //used in Database and Join
static const int LITERAL_FIRST_ID = 2 * 1000*1000*1000; static const unsigned LITERAL_FIRST_ID = 2 * 1000*1000*1000;
//static const int LITERAL_FIRST_ID = 2 * 1000*1000*1000;
//initial transfer buffer size in Tree/ and Stream/ //initial transfer buffer size in Tree/ and Stream/
static const unsigned TRANSFER_SIZE = 1 << 20; //1M static const unsigned TRANSFER_SIZE = 1 << 20; //1M

2
api/cpp/src/Makefile
View File

@ -1,5 +1,5 @@
#CC=g++ #CC=g++
CC=ccache g++ CC=ccache g++ -std=c++11
lib_dir=../lib/ lib_dir=../lib/
socket_obj_dir=../../../.objs/ socket_obj_dir=../../../.objs/

10
makefile
View File

@ -35,7 +35,7 @@
#compile parameters #compile parameters
CC = ccache g++ -std=c++11 CC = ccache g++
#CC = g++ #CC = g++
#the optimazition level of gcc/g++ #the optimazition level of gcc/g++
@ -43,11 +43,11 @@ CC = ccache g++ -std=c++11
#NOTICE: -O2 is recommended, while -O3 is dangerous #NOTICE: -O2 is recommended, while -O3 is dangerous
#when developing, not use -O because it will disturb the normal #when developing, not use -O because it will disturb the normal
#routine. use it for test and release. #routine. use it for test and release.
CFLAGS = -c -Wall -O2 CFLAGS = -c -Wall -O2 -pthread -std=c++11
EXEFLAG = -O2 EXEFLAG = -O2 -pthread -std=c++11
#-coverage #-coverage
#CFLAGS = -c -Wall -O2 -pthread #CFLAGS = -c -Wall -O2 -pthread -g
#EXEFLAG = -O2 -pthread #EXEFLAG = -O2 -pthread -g
#add -lreadline -ltermcap if using readline or objs contain readline #add -lreadline -ltermcap if using readline or objs contain readline
library = -ltermcap -lreadline -L./lib -L/usr/local/lib -lantlr -lgcov -lboost_filesystem -lboost_system -lboost_regex -lpthread -I/usr/local/include/boost library = -ltermcap -lreadline -L./lib -L/usr/local/lib -lantlr -lgcov -lboost_filesystem -lboost_system -lboost_regex -lpthread -I/usr/local/include/boost