opensci
/
gStore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'qinzongyue' into dev 

merge modifications of qinzongyue to indices
This commit is contained in:
bookug 2018-10-19 13:17:54 +08:00
parent f2c7e97655 0f0eb86ba9
commit a5e169afab
12 changed files with 124 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
Database/Database.cpp
View File

@ -51,7 +51,7 @@ Database::Database()
this->if_loaded = false; this->if_loaded = false;
this->trie = NULL; // this->trie = NULL;
//this->resetIDinfo(); //this->resetIDinfo();
this->initIDinfo(); this->initIDinfo();
@ -103,7 +103,7 @@ Database::Database(string _name)
this->query_cache = new QueryCache(); this->query_cache = new QueryCache();
this->trie = NULL; // this->trie = NULL;
//this->resetIDinfo(); //this->resetIDinfo();
this->initIDinfo(); this->initIDinfo();
@ -685,8 +685,8 @@ Database::load()
return false; return false;
} }
if(!(this->kvstore)->load_trie()) // if(!(this->kvstore)->load_trie())
return false; // return false;
//NOTICE: we should also run some heavy work in the main thread //NOTICE: we should also run some heavy work in the main thread
this->stringindex->load(); this->stringindex->load();
@ -736,17 +736,17 @@ Database::load()
//this->warmUp(); //this->warmUp();
//DEBUG:the warmUp() calls query(), which will also output results, this is not we want //DEBUG:the warmUp() calls query(), which will also output results, this is not we want
// Load trie // Load trie (trie has been removed)
if (trie != NULL) // if (trie != NULL)
delete trie; // delete trie;
trie = new Trie; // trie = new Trie;
string dictionary_path = store_path + "/dictionary.dc"; // string dictionary_path = store_path + "/dictionary.dc";
if (!trie->LoadTrie(dictionary_path)) // if (!trie->LoadTrie(dictionary_path))
{ // {
return false; // return false;
} // }
this->if_loaded = true; this->if_loaded = true;
@ -1259,11 +1259,11 @@ Database::unload()
this->if_loaded = false; this->if_loaded = false;
this->clear_update_log(); this->clear_update_log();
if (this->trie != NULL) // if (this->trie != NULL)
{ // {
delete this->trie; // delete this->trie;
trie = NULL; // trie = NULL;
} // }
return true; return true;
} }
@ -1395,7 +1395,7 @@ Database::get_query_parse_lock()
int int
Database::query(const string _query, ResultSet& _result_set, FILE* _fp) Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
{ {
string dictionary_store_path = this->store_path + "/dictionary.dc"; // string dictionary_store_path = this->store_path + "/dictionary.dc";
GeneralEvaluation general_evaluation(this->vstree, this->kvstore, this->stringindex, this->query_cache, this->pre2num, this->limitID_predicate, this->limitID_literal,this->limitID_entity); GeneralEvaluation general_evaluation(this->vstree, this->kvstore, this->stringindex, this->query_cache, this->pre2num, this->limitID_predicate, this->limitID_literal,this->limitID_entity);
@ -1444,16 +1444,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
long tv_bfget = Util::get_cur_time(); long tv_bfget = Util::get_cur_time();
//NOTICE: this lock lock ensures that StringIndex is visited sequentially //NOTICE: this lock lock ensures that StringIndex is visited sequentially
this->getFinalResult_lock.lock(); this->getFinalResult_lock.lock();
if (trie == NULL) // if (trie == NULL)
{ // {
trie = new Trie; // trie = new Trie;
string dictionary_path = this->store_path + "/dictionary.dc"; // string dictionary_path = this->store_path + "/dictionary.dc";
if (!trie->LoadTrie(dictionary_path)) // if (!trie->LoadTrie(dictionary_path))
{ // {
exit(0); // exit(0);
} // }
trie->LoadDictionary(); // trie->LoadDictionary();
} // }
general_evaluation.getFinalResult(_result_set); general_evaluation.getFinalResult(_result_set);
this->getFinalResult_lock.unlock(); this->getFinalResult_lock.unlock();
long tv_afget = Util::get_cur_time(); long tv_afget = Util::get_cur_time();
@ -1485,16 +1485,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
TripleWithObjType *update_triple = NULL; TripleWithObjType *update_triple = NULL;
TYPE_TRIPLE_NUM update_triple_num = 0; TYPE_TRIPLE_NUM update_triple_num = 0;
if (trie == NULL) // if (trie == NULL)
{ // {
trie = new Trie; // trie = new Trie;
string dictionary_path = this->store_path + "/dictionary.dc"; // string dictionary_path = this->store_path + "/dictionary.dc";
if (!trie->LoadTrie(dictionary_path)) // if (!trie->LoadTrie(dictionary_path))
{ // {
exit(0); // exit(0);
} // }
trie->LoadDictionary(); // trie->LoadDictionary();
} // }
if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Insert_Data || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Data) if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Insert_Data || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Data)
@ -1519,7 +1519,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
update_pattern.sub_group_pattern[i].pattern.object.value, object_type); update_pattern.sub_group_pattern[i].pattern.object.value, object_type);
// Compress // Compress
update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); // update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE);
} }
else else
{ {
@ -1547,7 +1547,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getDeletePatterns(), update_triple, update_triple_num); general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getDeletePatterns(), update_triple, update_triple_num);
for(int i = 0; i < update_triple_num; i++) for(int i = 0; i < update_triple_num; i++)
{ {
update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); // update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE);
} }
success_num = remove(update_triple, update_triple_num); success_num = remove(update_triple, update_triple_num);
} }
@ -1556,7 +1556,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getInsertPatterns(), update_triple, update_triple_num); general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getInsertPatterns(), update_triple, update_triple_num);
for(int i = 0; i < update_triple_num; i++) for(int i = 0; i < update_triple_num; i++)
{ {
update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); // update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE);
} }
success_num = insert(update_triple, update_triple_num); success_num = insert(update_triple, update_triple_num);
} }
@ -2413,7 +2413,7 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file)
(this->kvstore)->open_id2predicate(KVstore::CREATE_MODE); (this->kvstore)->open_id2predicate(KVstore::CREATE_MODE);
(this->kvstore)->open_literal2id(KVstore::CREATE_MODE); (this->kvstore)->open_literal2id(KVstore::CREATE_MODE);
(this->kvstore)->open_id2literal(KVstore::CREATE_MODE); (this->kvstore)->open_id2literal(KVstore::CREATE_MODE);
(this->kvstore)->load_trie(); // (this->kvstore)->load_trie();
} }
//Util::logging("finish initial sub2id_pre2id_obj2id"); //Util::logging("finish initial sub2id_pre2id_obj2id");
@ -2457,19 +2457,19 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file)
RDFParser _parser(_fin); RDFParser _parser(_fin);
// Initialize trie // Initialize trie
string dictionary_store_path = this->store_path + "/dictionary.dc"; // string dictionary_store_path = this->store_path + "/dictionary.dc";
this->trie = new Trie (_rdf_file, dictionary_store_path); // this->trie = new Trie (_rdf_file, dictionary_store_path);
if (!trie->isInitialized()) // if (!trie->isInitialized())
{ // {
cout << "Fail to initialize trie" << endl; // cout << "Fail to initialize trie" << endl;
exit(0); // exit(0);
} // }
if(!trie->WriteDown()) // if(!trie->WriteDown())
{ // {
cout << "Fail to write down dictionary" << endl; // cout << "Fail to write down dictionary" << endl;
exit(0); // exit(0);
} // }
//Util::logging("==> while(true)"); //Util::logging("==> while(true)");
while (true) while (true)
@ -2515,8 +2515,8 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file)
//We need a thread pool! //We need a thread pool!
// Compress triple begin // Compress triple begin
TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE); // TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE);
triple_array[i] = compressed_triple; // triple_array[i] = compressed_triple;
//Compress triple end //Compress triple end
// For subject // For subject
@ -3036,11 +3036,11 @@ Database::insert(std::string _rdf_file, bool _is_restore)
} }
//Compress triple here //Compress triple here
for(int i = 0; i < parse_triple_num; i++) // for(int i = 0; i < parse_triple_num; i++)
{ // {
TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); // TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE);
triple_array[i] = compressed_triple; // triple_array[i] = compressed_triple;
} // }
//Process the Triple one by one //Process the Triple one by one
success_num += this->insert(triple_array, parse_triple_num, _is_restore); success_num += this->insert(triple_array, parse_triple_num, _is_restore);
@ -3136,11 +3136,11 @@ Database::remove(std::string _rdf_file, bool _is_restore)
} }
//Compress triple //Compress triple
for(int i = 0; i < parse_triple_num; i++) // for(int i = 0; i < parse_triple_num; i++)
{ // {
TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); // TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE);
triple_array[i] = compressed_triple; // triple_array[i] = compressed_triple;
} // }
success_num += this->remove(triple_array, parse_triple_num, _is_restore); success_num += this->remove(triple_array, parse_triple_num, _is_restore);

2
Database/Database.h
View File

@ -155,7 +155,7 @@ private:
QueryCache *query_cache; QueryCache *query_cache;
Trie *trie; //Trie *trie;
void setStringBuffer(); void setStringBuffer();
void warmUp(); void warmUp();

4
KVstore/ISArray/ISArray.cpp
View File

@ -16,7 +16,6 @@ ISArray::ISArray()
dir_path = ""; dir_path = "";
ISfile_name = ""; ISfile_name = "";
BM = NULL; BM = NULL;
CurKeyNum = 0;
CurEntryNum = 0; CurEntryNum = 0;
CurCacheSize = 0; CurCacheSize = 0;
CurEntryNumChange = false; CurEntryNumChange = false;
@ -55,7 +54,6 @@ ISArray::ISArray(string _dir_path, string _filename, string mode, unsigned long
// temp is the smallest number >= _key_num and mod SET_KEY_INC = 0 // temp is the smallest number >= _key_num and mod SET_KEY_INC = 0
unsigned temp = ((_key_num + (1 << 10) - 1) >> 10) << 10; unsigned temp = ((_key_num + (1 << 10) - 1) >> 10) << 10;
CurKeyNum = 0;
CurEntryNum = max(temp, SETKEYNUM); CurEntryNum = max(temp, SETKEYNUM);
CurEntryNumChange = true; CurEntryNumChange = true;
@ -341,7 +339,6 @@ ISArray::insert(unsigned _key, char *_str, unsigned _len)
} }
bool Alloc = false; bool Alloc = false;
CurKeyNum++;
//if (CurKeyNum >= CurEntryNum) // need to realloc //if (CurKeyNum >= CurEntryNum) // need to realloc
if (_key >= CurEntryNum) if (_key >= CurEntryNum)
{ {
@ -391,7 +388,6 @@ ISArray::remove(unsigned _key)
return false; return false;
} }
CurKeyNum--;
unsigned store = array[_key].getStore(); unsigned store = array[_key].getStore();
BM->FreeBlocks(store); BM->FreeBlocks(store);

1
KVstore/ISArray/ISArray.h
View File

@ -35,7 +35,6 @@ private:
string dir_path; string dir_path;
ISBlockManager *BM; ISBlockManager *BM;
unsigned CurEntryNum; // how many entries are available unsigned CurEntryNum; // how many entries are available
unsigned CurKeyNum; // how many keys are stored
bool CurEntryNumChange; bool CurEntryNumChange;
//Cache //Cache

8
KVstore/ISArray/ISBlockManager.cpp
View File

@ -335,8 +335,10 @@ ISBlockManager::FreeBlocks(const unsigned index)
{ {
cur_index = it->first; cur_index = it->first;
curlen += it->second; curlen += it->second;
auto tmp_key = it->second, tmp_val = it->first;
index_len_map.erase(it); index_len_map.erase(it);
len_index_map.erase(make_pair(it->second,it->first)); //len_index_map.erase(make_pair(it->second,it->first));
len_index_map.erase(make_pair(tmp_key, tmp_val));
} }
} }
@ -346,8 +348,10 @@ ISBlockManager::FreeBlocks(const unsigned index)
if (curlen + cur_index == it->first) // block after is free if (curlen + cur_index == it->first) // block after is free
{ {
curlen += it->second; curlen += it->second;
auto tmp_key = it->second, tmp_val = it->first;
index_len_map.erase(it); index_len_map.erase(it);
len_index_map.erase(make_pair(it->second,it->first)); //len_index_map.erase(make_pair(it->second,it->first));
len_index_map.erase(make_pair(tmp_key, tmp_val));
} }
} }

4
KVstore/IVArray/IVArray.cpp
View File

@ -16,7 +16,6 @@ IVArray::IVArray()
dir_path = ""; dir_path = "";
IVfile_name = ""; IVfile_name = "";
BM = NULL; BM = NULL;
CurKeyNum = 0;
CurEntryNum = 0; CurEntryNum = 0;
CurCacheSize = 0; CurCacheSize = 0;
CurEntryNumChange = false; CurEntryNumChange = false;
@ -63,7 +62,6 @@ IVArray::IVArray(string _dir_path, string _filename, string mode, unsigned long
// temp is the smallest number >= _key_num and mod SET_KEY_INC = 0 // temp is the smallest number >= _key_num and mod SET_KEY_INC = 0
unsigned temp = ((_key_num + (1 << 10) - 1) >> 10) << 10; unsigned temp = ((_key_num + (1 << 10) - 1) >> 10) << 10;
CurKeyNum = 0;
CurEntryNum = max(temp, SETKEYNUM); CurEntryNum = max(temp, SETKEYNUM);
CurEntryNumChange = true; CurEntryNumChange = true;
@ -346,7 +344,6 @@ IVArray::insert(unsigned _key, char *_str, unsigned _len)
return false; return false;
} }
CurKeyNum++;
//if (CurKeyNum >= CurEntryNum) // need to realloc //if (CurKeyNum >= CurEntryNum) // need to realloc
if (_key >= CurEntryNum) if (_key >= CurEntryNum)
{ {
@ -402,7 +399,6 @@ IVArray::remove(unsigned _key)
return false; return false;
} }
CurKeyNum--;
unsigned store = array[_key].getStore(); unsigned store = array[_key].getStore();
BM->FreeBlocks(store); BM->FreeBlocks(store);

1
KVstore/IVArray/IVArray.h
View File

@ -36,7 +36,6 @@ private:
string dir_path; string dir_path;
IVBlockManager *BM; IVBlockManager *BM;
unsigned CurEntryNum; // how many entries are available unsigned CurEntryNum; // how many entries are available
unsigned CurKeyNum; // how many keys are stored
bool CurEntryNumChange; bool CurEntryNumChange;
//Cache //Cache

4
KVstore/IVArray/IVBlockManager.cpp
View File

@ -335,8 +335,10 @@ IVBlockManager::FreeBlocks(const unsigned index)
{ {
cur_index = it->first; cur_index = it->first;
curlen += it->second; curlen += it->second;
auto tmp_key = it->second, tmp_val = it->first;
index_len_map.erase(it); index_len_map.erase(it);
len_index_map.erase(make_pair(it->second,it->first)); //len_index_map.erase(make_pair(it->second,it->first));
len_index_map.erase(make_pair(tmp_key, tmp_val));
} }
} }

50
KVstore/KVstore.cpp
View File

@ -16,8 +16,8 @@ KVstore::KVstore(string _store_path)
{ {
this->store_path = _store_path; this->store_path = _store_path;
this->dictionary_store_path = _store_path + "/../dictionary.dc"; // this->dictionary_store_path = _store_path + "/../dictionary.dc";
this->trie = NULL; //this->trie = NULL;
this->entity2id = NULL; this->entity2id = NULL;
this->id2entity = NULL; this->id2entity = NULL;
@ -97,9 +97,9 @@ KVstore::release()
delete this->objID2values_literal; delete this->objID2values_literal;
this->objID2values_literal = NULL; this->objID2values_literal = NULL;
if (trie != NULL) //if (trie != NULL)
delete this->trie; // delete this->trie;
this->trie = NULL; //this->trie = NULL;
} }
void void
@ -120,18 +120,18 @@ KVstore::open()
this->open_objID2values(KVstore::READ_WRITE_MODE); this->open_objID2values(KVstore::READ_WRITE_MODE);
this->open_preID2values(KVstore::READ_WRITE_MODE); this->open_preID2values(KVstore::READ_WRITE_MODE);
this->trie = new Trie; //this->trie = new Trie;
trie->LoadTrie(dictionary_store_path); //trie->LoadTrie(dictionary_store_path);
} }
bool //bool
KVstore::load_trie() //KVstore::load_trie()
{ //{
if (trie != NULL) return true; // if (trie != NULL) return true;
trie = new Trie; // trie = new Trie;
return trie->LoadTrie(dictionary_store_path); // return trie->LoadTrie(dictionary_store_path);
} //}
string string
KVstore::getStringByID(TYPE_ENTITY_LITERAL_ID _id) KVstore::getStringByID(TYPE_ENTITY_LITERAL_ID _id)
@ -1099,7 +1099,7 @@ KVstore::subIDByEntity(string _entity)
//_entity will not be released befor ethis function ends //_entity will not be released befor ethis function ends
//so _entity.c_str() is a valid const char* //so _entity.c_str() is a valid const char*
//this->load_trie(); //this->load_trie();
_entity = trie->Compress(_entity); //_entity = trie->Compress(_entity);
//return this->entity2id->remove(_entity.c_str(), _entity.length()); //return this->entity2id->remove(_entity.c_str(), _entity.length());
return this->removeKey(this->entity2id, _entity.c_str(), _entity.length()); return this->removeKey(this->entity2id, _entity.c_str(), _entity.length());
} }
@ -1108,7 +1108,7 @@ TYPE_ENTITY_LITERAL_ID
KVstore::getIDByEntity(string _entity) const KVstore::getIDByEntity(string _entity) const
{ {
//this->load_trie(); //this->load_trie();
_entity = trie->Compress(_entity); //_entity = trie->Compress(_entity);
return this->getIDByStr(this->entity2id, _entity.c_str(), _entity.length()); return this->getIDByStr(this->entity2id, _entity.c_str(), _entity.length());
} }
@ -1121,7 +1121,7 @@ KVstore::setIDByEntity(string _entity, TYPE_ENTITY_LITERAL_ID _id)
char* str = new char[len]; char* str = new char[len];
//this->load_trie(); //this->load_trie();
_entity = trie->Compress(_entity); //_entity = trie->Compress(_entity);
memcpy(str, _entity.c_str(), len); memcpy(str, _entity.c_str(), len);
return this->addValueByKey(this->entity2id, str, len, _id); return this->addValueByKey(this->entity2id, str, len, _id);
} }
@ -1196,7 +1196,7 @@ KVstore::getEntityByID(TYPE_ENTITY_LITERAL_ID _id) const
//cout << _tmp << endl; //cout << _tmp << endl;
string _ret = string(_tmp, _len); string _ret = string(_tmp, _len);
_ret = trie->Uncompress(_ret, _ret.length()); // _ret = trie->Uncompress(_ret, _ret.length());
delete []_tmp; delete []_tmp;
// cout << "string is: " << _ret << endl; // cout << "string is: " << _ret << endl;
// string test1 = _ret; // string test1 = _ret;
@ -1260,7 +1260,7 @@ bool
KVstore::subIDByPredicate(string _predicate) KVstore::subIDByPredicate(string _predicate)
{ {
//this->load_trie(); //this->load_trie();
_predicate = trie->Compress(_predicate); // _predicate = trie->Compress(_predicate);
//return this->predicate2id->remove(_predicate.c_str(), _predicate.length()); //return this->predicate2id->remove(_predicate.c_str(), _predicate.length());
return this->removeKey(this->predicate2id, _predicate.c_str(), _predicate.length()); return this->removeKey(this->predicate2id, _predicate.c_str(), _predicate.length());
} }
@ -1269,7 +1269,7 @@ TYPE_PREDICATE_ID
KVstore::getIDByPredicate(string _predicate) const KVstore::getIDByPredicate(string _predicate) const
{ {
//this->load_trie(); //this->load_trie();
_predicate = trie->Compress(_predicate); // _predicate = trie->Compress(_predicate);
return this->getIDByStr(this->predicate2id, _predicate.c_str(), _predicate.length()); return this->getIDByStr(this->predicate2id, _predicate.c_str(), _predicate.length());
} }
@ -1282,7 +1282,7 @@ KVstore::setIDByPredicate(string _predicate, TYPE_PREDICATE_ID _id)
char* str = new char[len]; char* str = new char[len];
//this->load_trie(); //this->load_trie();
_predicate = trie->Compress(_predicate); // _predicate = trie->Compress(_predicate);
memcpy(str, _predicate.c_str(), len); memcpy(str, _predicate.c_str(), len);
return this->addValueByKey(this->predicate2id, str, len, _id); return this->addValueByKey(this->predicate2id, str, len, _id);
} }
@ -1402,7 +1402,7 @@ bool
KVstore::subIDByLiteral(string _literal) KVstore::subIDByLiteral(string _literal)
{ {
//this->load_trie(); //this->load_trie();
_literal = trie->Compress(_literal); // _literal = trie->Compress(_literal);
//return this->literal2id->remove(_literal.c_str(), _literal.length()); //return this->literal2id->remove(_literal.c_str(), _literal.length());
return this->removeKey(this->literal2id, _literal.c_str(), _literal.length()); return this->removeKey(this->literal2id, _literal.c_str(), _literal.length());
} }
@ -1411,7 +1411,7 @@ TYPE_ENTITY_LITERAL_ID
KVstore::getIDByLiteral(string _literal) const KVstore::getIDByLiteral(string _literal) const
{ {
//this->load_trie(); //this->load_trie();
_literal = trie->Compress(_literal); // _literal = trie->Compress(_literal);
return this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length()); return this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length());
//TYPE_ENTITY_LITERAL_ID id = this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length()); //TYPE_ENTITY_LITERAL_ID id = this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length());
//if(id != INVALID) //if(id != INVALID)
@ -1428,7 +1428,7 @@ KVstore::setIDByLiteral(string _literal, TYPE_ENTITY_LITERAL_ID _id)
//this->load_trie(); //this->load_trie();
int len = _literal.length(); int len = _literal.length();
char* str = new char[len]; char* str = new char[len];
_literal = trie->Compress(_literal); // _literal = trie->Compress(_literal);
memcpy(str, _literal.c_str(), len); memcpy(str, _literal.c_str(), len);
return this->addValueByKey(this->literal2id, str, len, _id); return this->addValueByKey(this->literal2id, str, len, _id);
@ -1493,7 +1493,7 @@ KVstore::getLiteralByID(TYPE_ENTITY_LITERAL_ID _id) const
} }
string _ret = string(_tmp, _len); string _ret = string(_tmp, _len);
delete []_tmp; delete []_tmp;
_ret = trie->Uncompress(_ret, _ret.length()); // _ret = trie->Uncompress(_ret, _ret.length());
return _ret; return _ret;
} }

8
KVstore/KVstore.h
View File

@ -12,7 +12,7 @@
#include "../Util/Util.h" #include "../Util/Util.h"
#include "../Util/VList.h" #include "../Util/VList.h"
#include "Tree.h" #include "Tree.h"
#include "../Trie/Trie.h" //#include "../Trie/Trie.h"
#include "IVArray/IVArray.h" #include "IVArray/IVArray.h"
#include "ISArray/ISArray.h" #include "ISArray/ISArray.h"
@ -169,14 +169,14 @@ public:
//for so2p //for so2p
bool getpreIDlistBysubIDobjID(TYPE_ENTITY_LITERAL_ID _subID, TYPE_ENTITY_LITERAL_ID _objID, unsigned*& _preidlist, unsigned& _list_len, bool _no_duplicate = false) const; bool getpreIDlistBysubIDobjID(TYPE_ENTITY_LITERAL_ID _subID, TYPE_ENTITY_LITERAL_ID _objID, unsigned*& _preidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool load_trie(); //bool load_trie();
private: private:
std::string store_path; std::string store_path;
std::string dictionary_store_path; // std::string dictionary_store_path;
Trie *trie; //Trie *trie;
SITree* entity2id; SITree* entity2id;
//ISTree* id2entity; //ISTree* id2entity;

18
StringIndex/StringIndex.cpp
View File

@ -81,7 +81,7 @@ void StringIndexFile::load()
this->empty_offset = max(this->empty_offset, (*this->index_table)[i].offset + (long)(*this->index_table)[i].length); this->empty_offset = max(this->empty_offset, (*this->index_table)[i].offset + (long)(*this->index_table)[i].length);
} }
trie->LoadTrie(dictionary_path); // trie->LoadTrie(dictionary_path);
} }
bool StringIndexFile::randomAccess(unsigned id, string *str, bool real) bool StringIndexFile::randomAccess(unsigned id, string *str, bool real)
@ -110,10 +110,10 @@ bool StringIndexFile::randomAccess(unsigned id, string *str, bool real)
//cout<<"check: "<<*str<<endl; //cout<<"check: "<<*str<<endl;
//} //}
if (real) // if (real)
{ // {
*str = trie->Uncompress(*str, str->length());//Uncompresss // *str = trie->Uncompress(*str, str->length());//Uncompresss
} // }
//if(id == 9) //if(id == 9)
//{ //{
//cout<<"check: "<<*str<<endl; //cout<<"check: "<<*str<<endl;
@ -194,8 +194,8 @@ void StringIndexFile::trySequenceAccess(bool real)
this->buffer[length] = '\0'; this->buffer[length] = '\0';
*this->request[pos].str = string(this->buffer); *this->request[pos].str = string(this->buffer);
*this->request[pos].str = trie->Uncompress( // *this->request[pos].str = trie->Uncompress(
*this->request[pos].str, this->request[pos].str->length()); // *this->request[pos].str, this->request[pos].str->length());
pos++; pos++;
} }
@ -216,8 +216,8 @@ void StringIndexFile::trySequenceAccess(bool real)
this->buffer[length] = '\0'; this->buffer[length] = '\0';
*this->request[pos].str += string(this->buffer); *this->request[pos].str += string(this->buffer);
*this->request[pos].str = trie->Uncompress( // *this->request[pos].str = trie->Uncompress(
*this->request[pos].str, this->request[pos].str->length()); // *this->request[pos].str, this->request[pos].str->length());
pos++; pos++;
while (pos < (int)this->request.size() && this->request[pos - 1].offset == this->request[pos].offset) while (pos < (int)this->request.size() && this->request[pos - 1].offset == this->request[pos].offset)

18
StringIndex/StringIndex.h
View File

@ -52,10 +52,10 @@ class StringIndexFile
}; };
std::vector<AccessRequest> request; std::vector<AccessRequest> request;
std::string dictionary_path; // std::string dictionary_path;
public: public:
Trie *trie; //Trie *trie;
StringIndexFile(StringIndexFileType _type, std::string _dir, unsigned _num):type(_type), num(_num), empty_offset(0), index_file(NULL), value_file(NULL), buffer_size(0), buffer(NULL) StringIndexFile(StringIndexFileType _type, std::string _dir, unsigned _num):type(_type), num(_num), empty_offset(0), index_file(NULL), value_file(NULL), buffer_size(0), buffer(NULL)
{ {
@ -67,8 +67,8 @@ class StringIndexFile
this->loc = _dir + "/predicate_"; this->loc = _dir + "/predicate_";
this->index_table = new std::vector<IndexInfo>; this->index_table = new std::vector<IndexInfo>;
dictionary_path = _dir + "/../dictionary.dc"; //dictionary_path = _dir + "/../dictionary.dc";
trie = new Trie; //trie = new Trie;
} }
~StringIndexFile() ~StringIndexFile()
{ {
@ -79,8 +79,8 @@ class StringIndexFile
if (this->buffer != NULL) if (this->buffer != NULL)
delete[] this->buffer; delete[] this->buffer;
delete this->index_table; delete this->index_table;
if (this->trie != NULL) // if (this->trie != NULL)
delete trie; // delete trie;
} }
void clear() void clear()
@ -88,7 +88,7 @@ class StringIndexFile
this->index_file = NULL; this->index_file = NULL;
this->value_file = NULL; this->value_file = NULL;
this->index_table = NULL; this->index_table = NULL;
this->trie = NULL; // this->trie = NULL;
} }
void setNum(unsigned _num); void setNum(unsigned _num);
@ -132,12 +132,12 @@ class StringIndex
unsigned literal_buffer_size; unsigned literal_buffer_size;
public: public:
Trie *trie; // Trie *trie;
StringIndex(std::string _dir, unsigned _entity_num = 0, unsigned _literal_num = 0, unsigned _predicate_num = 0): StringIndex(std::string _dir, unsigned _entity_num = 0, unsigned _literal_num = 0, unsigned _predicate_num = 0):
entity(StringIndexFile::Entity, _dir, _entity_num), literal(StringIndexFile::Literal, _dir, _literal_num), predicate(StringIndexFile::Predicate, _dir, _predicate_num) entity(StringIndexFile::Entity, _dir, _entity_num), literal(StringIndexFile::Literal, _dir, _literal_num), predicate(StringIndexFile::Predicate, _dir, _predicate_num)
{ {
trie = entity.trie; // trie = entity.trie;
} }
void clear() void clear()