From 15521f61320e52852267183dc544438aa332e233 Mon Sep 17 00:00:00 2001 From: Zongyue Date: Fri, 19 Oct 2018 04:57:06 +0800 Subject: [PATCH 1/3] Fix: remove trie and fix bugs in IS(IV)BlockManager.cpp --- Database/Database.cpp | 134 ++++++++++++++--------------- Database/Database.h | 2 +- KVstore/ISArray/ISBlockManager.cpp | 8 +- KVstore/IVArray/IVBlockManager.cpp | 4 +- KVstore/KVstore.cpp | 50 +++++------ KVstore/KVstore.h | 8 +- StringIndex/StringIndex.cpp | 18 ++-- StringIndex/StringIndex.h | 18 ++-- 8 files changed, 124 insertions(+), 118 deletions(-) diff --git a/Database/Database.cpp b/Database/Database.cpp index ed8f27c..7189b7b 100644 --- a/Database/Database.cpp +++ b/Database/Database.cpp @@ -51,7 +51,7 @@ Database::Database() this->if_loaded = false; - this->trie = NULL; +// this->trie = NULL; //this->resetIDinfo(); this->initIDinfo(); @@ -103,7 +103,7 @@ Database::Database(string _name) this->query_cache = new QueryCache(); - this->trie = NULL; +// this->trie = NULL; //this->resetIDinfo(); this->initIDinfo(); @@ -685,8 +685,8 @@ Database::load() return false; } - if(!(this->kvstore)->load_trie()) - return false; +// if(!(this->kvstore)->load_trie()) +// return false; //NOTICE: we should also run some heavy work in the main thread this->stringindex->load(); @@ -736,17 +736,17 @@ Database::load() //this->warmUp(); //DEBUG:the warmUp() calls query(), which will also output results, this is not we want - // Load trie + // Load trie (trie has been removed) - if (trie != NULL) - delete trie; - trie = new Trie; +// if (trie != NULL) +// delete trie; +// trie = new Trie; - string dictionary_path = store_path + "/dictionary.dc"; - if (!trie->LoadTrie(dictionary_path)) - { - return false; - } +// string dictionary_path = store_path + "/dictionary.dc"; +// if (!trie->LoadTrie(dictionary_path)) +// { +// return false; +// } this->if_loaded = true; @@ -1259,11 +1259,11 @@ Database::unload() this->if_loaded = false; this->clear_update_log(); - if (this->trie != NULL) - { - delete this->trie; - trie = NULL; - } +// if (this->trie != NULL) +// { +// delete this->trie; +// trie = NULL; +// } return true; } @@ -1395,7 +1395,7 @@ Database::get_query_parse_lock() int Database::query(const string _query, ResultSet& _result_set, FILE* _fp) { - string dictionary_store_path = this->store_path + "/dictionary.dc"; +// string dictionary_store_path = this->store_path + "/dictionary.dc"; GeneralEvaluation general_evaluation(this->vstree, this->kvstore, this->stringindex, this->query_cache, this->pre2num, this->limitID_predicate, this->limitID_literal,this->limitID_entity); @@ -1444,16 +1444,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) long tv_bfget = Util::get_cur_time(); //NOTICE: this lock lock ensures that StringIndex is visited sequentially this->getFinalResult_lock.lock(); - if (trie == NULL) - { - trie = new Trie; - string dictionary_path = this->store_path + "/dictionary.dc"; - if (!trie->LoadTrie(dictionary_path)) - { - exit(0); - } - trie->LoadDictionary(); - } +// if (trie == NULL) +// { +// trie = new Trie; +// string dictionary_path = this->store_path + "/dictionary.dc"; +// if (!trie->LoadTrie(dictionary_path)) +// { +// exit(0); +// } +// trie->LoadDictionary(); +// } general_evaluation.getFinalResult(_result_set); this->getFinalResult_lock.unlock(); long tv_afget = Util::get_cur_time(); @@ -1485,16 +1485,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) TripleWithObjType *update_triple = NULL; TYPE_TRIPLE_NUM update_triple_num = 0; - if (trie == NULL) - { - trie = new Trie; - string dictionary_path = this->store_path + "/dictionary.dc"; - if (!trie->LoadTrie(dictionary_path)) - { - exit(0); - } - trie->LoadDictionary(); - } +// if (trie == NULL) +// { +// trie = new Trie; +// string dictionary_path = this->store_path + "/dictionary.dc"; +// if (!trie->LoadTrie(dictionary_path)) +// { +// exit(0); +// } +// trie->LoadDictionary(); +// } if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Insert_Data || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Data) @@ -1519,7 +1519,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) update_pattern.sub_group_pattern[i].pattern.object.value, object_type); // Compress - update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); +// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); } else { @@ -1547,7 +1547,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getDeletePatterns(), update_triple, update_triple_num); for(int i = 0; i < update_triple_num; i++) { - update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); +// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); } success_num = remove(update_triple, update_triple_num); } @@ -1556,7 +1556,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getInsertPatterns(), update_triple, update_triple_num); for(int i = 0; i < update_triple_num; i++) { - update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); +// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); } success_num = insert(update_triple, update_triple_num); } @@ -2413,7 +2413,7 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file) (this->kvstore)->open_id2predicate(KVstore::CREATE_MODE); (this->kvstore)->open_literal2id(KVstore::CREATE_MODE); (this->kvstore)->open_id2literal(KVstore::CREATE_MODE); - (this->kvstore)->load_trie(); +// (this->kvstore)->load_trie(); } //Util::logging("finish initial sub2id_pre2id_obj2id"); @@ -2457,19 +2457,19 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file) RDFParser _parser(_fin); // Initialize trie - string dictionary_store_path = this->store_path + "/dictionary.dc"; - this->trie = new Trie (_rdf_file, dictionary_store_path); - if (!trie->isInitialized()) - { - cout << "Fail to initialize trie" << endl; - exit(0); - } +// string dictionary_store_path = this->store_path + "/dictionary.dc"; +// this->trie = new Trie (_rdf_file, dictionary_store_path); +// if (!trie->isInitialized()) +// { +// cout << "Fail to initialize trie" << endl; +// exit(0); +// } - if(!trie->WriteDown()) - { - cout << "Fail to write down dictionary" << endl; - exit(0); - } +// if(!trie->WriteDown()) +// { +// cout << "Fail to write down dictionary" << endl; +// exit(0); +// } //Util::logging("==> while(true)"); while (true) @@ -2515,8 +2515,8 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file) //We need a thread pool! // Compress triple begin - TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE); - triple_array[i] = compressed_triple; +// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE); +// triple_array[i] = compressed_triple; //Compress triple end // For subject @@ -3036,11 +3036,11 @@ Database::insert(std::string _rdf_file, bool _is_restore) } //Compress triple here - for(int i = 0; i < parse_triple_num; i++) - { - TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); - triple_array[i] = compressed_triple; - } +// for(int i = 0; i < parse_triple_num; i++) +// { +// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); +// triple_array[i] = compressed_triple; +// } //Process the Triple one by one success_num += this->insert(triple_array, parse_triple_num, _is_restore); @@ -3136,11 +3136,11 @@ Database::remove(std::string _rdf_file, bool _is_restore) } //Compress triple - for(int i = 0; i < parse_triple_num; i++) - { - TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); - triple_array[i] = compressed_triple; - } +// for(int i = 0; i < parse_triple_num; i++) +// { +// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); +// triple_array[i] = compressed_triple; +// } success_num += this->remove(triple_array, parse_triple_num, _is_restore); diff --git a/Database/Database.h b/Database/Database.h index cca19d1..61ba013 100644 --- a/Database/Database.h +++ b/Database/Database.h @@ -155,7 +155,7 @@ private: QueryCache *query_cache; - Trie *trie; + //Trie *trie; void setStringBuffer(); void warmUp(); diff --git a/KVstore/ISArray/ISBlockManager.cpp b/KVstore/ISArray/ISBlockManager.cpp index 6ac9399..302e73b 100644 --- a/KVstore/ISArray/ISBlockManager.cpp +++ b/KVstore/ISArray/ISBlockManager.cpp @@ -335,8 +335,10 @@ ISBlockManager::FreeBlocks(const unsigned index) { cur_index = it->first; curlen += it->second; + auto tmp_key = it->second, tmp_val = it->first; index_len_map.erase(it); - len_index_map.erase(make_pair(it->second,it->first)); + //len_index_map.erase(make_pair(it->second,it->first)); + len_index_map.erase(make_pair(tmp_key, tmp_val)); } } @@ -346,8 +348,10 @@ ISBlockManager::FreeBlocks(const unsigned index) if (curlen + cur_index == it->first) // block after is free { curlen += it->second; + auto tmp_key = it->second, tmp_val = it->first; index_len_map.erase(it); - len_index_map.erase(make_pair(it->second,it->first)); + //len_index_map.erase(make_pair(it->second,it->first)); + len_index_map.erase(make_pair(tmp_key, tmp_val)); } } diff --git a/KVstore/IVArray/IVBlockManager.cpp b/KVstore/IVArray/IVBlockManager.cpp index 212e4ec..2cebe5b 100644 --- a/KVstore/IVArray/IVBlockManager.cpp +++ b/KVstore/IVArray/IVBlockManager.cpp @@ -335,8 +335,10 @@ IVBlockManager::FreeBlocks(const unsigned index) { cur_index = it->first; curlen += it->second; + auto tmp_key = it->second, tmp_val = it->first; index_len_map.erase(it); - len_index_map.erase(make_pair(it->second,it->first)); + //len_index_map.erase(make_pair(it->second,it->first)); + len_index_map.erase(make_pair(tmp_key, tmp_val)); } } diff --git a/KVstore/KVstore.cpp b/KVstore/KVstore.cpp index b9db2b3..7343ec7 100644 --- a/KVstore/KVstore.cpp +++ b/KVstore/KVstore.cpp @@ -16,8 +16,8 @@ KVstore::KVstore(string _store_path) { this->store_path = _store_path; - this->dictionary_store_path = _store_path + "/../dictionary.dc"; - this->trie = NULL; +// this->dictionary_store_path = _store_path + "/../dictionary.dc"; + //this->trie = NULL; this->entity2id = NULL; this->id2entity = NULL; @@ -97,9 +97,9 @@ KVstore::release() delete this->objID2values_literal; this->objID2values_literal = NULL; - if (trie != NULL) - delete this->trie; - this->trie = NULL; + //if (trie != NULL) + // delete this->trie; + //this->trie = NULL; } void @@ -120,18 +120,18 @@ KVstore::open() this->open_objID2values(KVstore::READ_WRITE_MODE); this->open_preID2values(KVstore::READ_WRITE_MODE); - this->trie = new Trie; - trie->LoadTrie(dictionary_store_path); + //this->trie = new Trie; + //trie->LoadTrie(dictionary_store_path); } -bool -KVstore::load_trie() -{ - if (trie != NULL) return true; +//bool +//KVstore::load_trie() +//{ +// if (trie != NULL) return true; - trie = new Trie; - return trie->LoadTrie(dictionary_store_path); -} +// trie = new Trie; +// return trie->LoadTrie(dictionary_store_path); +//} string KVstore::getStringByID(TYPE_ENTITY_LITERAL_ID _id) @@ -1099,7 +1099,7 @@ KVstore::subIDByEntity(string _entity) //_entity will not be released befor ethis function ends //so _entity.c_str() is a valid const char* //this->load_trie(); - _entity = trie->Compress(_entity); + //_entity = trie->Compress(_entity); //return this->entity2id->remove(_entity.c_str(), _entity.length()); return this->removeKey(this->entity2id, _entity.c_str(), _entity.length()); } @@ -1108,7 +1108,7 @@ TYPE_ENTITY_LITERAL_ID KVstore::getIDByEntity(string _entity) const { //this->load_trie(); - _entity = trie->Compress(_entity); + //_entity = trie->Compress(_entity); return this->getIDByStr(this->entity2id, _entity.c_str(), _entity.length()); } @@ -1121,7 +1121,7 @@ KVstore::setIDByEntity(string _entity, TYPE_ENTITY_LITERAL_ID _id) char* str = new char[len]; //this->load_trie(); - _entity = trie->Compress(_entity); + //_entity = trie->Compress(_entity); memcpy(str, _entity.c_str(), len); return this->addValueByKey(this->entity2id, str, len, _id); } @@ -1196,7 +1196,7 @@ KVstore::getEntityByID(TYPE_ENTITY_LITERAL_ID _id) const //cout << _tmp << endl; string _ret = string(_tmp, _len); - _ret = trie->Uncompress(_ret, _ret.length()); +// _ret = trie->Uncompress(_ret, _ret.length()); delete []_tmp; // cout << "string is: " << _ret << endl; // string test1 = _ret; @@ -1260,7 +1260,7 @@ bool KVstore::subIDByPredicate(string _predicate) { //this->load_trie(); - _predicate = trie->Compress(_predicate); +// _predicate = trie->Compress(_predicate); //return this->predicate2id->remove(_predicate.c_str(), _predicate.length()); return this->removeKey(this->predicate2id, _predicate.c_str(), _predicate.length()); } @@ -1269,7 +1269,7 @@ TYPE_PREDICATE_ID KVstore::getIDByPredicate(string _predicate) const { //this->load_trie(); - _predicate = trie->Compress(_predicate); +// _predicate = trie->Compress(_predicate); return this->getIDByStr(this->predicate2id, _predicate.c_str(), _predicate.length()); } @@ -1282,7 +1282,7 @@ KVstore::setIDByPredicate(string _predicate, TYPE_PREDICATE_ID _id) char* str = new char[len]; //this->load_trie(); - _predicate = trie->Compress(_predicate); +// _predicate = trie->Compress(_predicate); memcpy(str, _predicate.c_str(), len); return this->addValueByKey(this->predicate2id, str, len, _id); } @@ -1402,7 +1402,7 @@ bool KVstore::subIDByLiteral(string _literal) { //this->load_trie(); - _literal = trie->Compress(_literal); +// _literal = trie->Compress(_literal); //return this->literal2id->remove(_literal.c_str(), _literal.length()); return this->removeKey(this->literal2id, _literal.c_str(), _literal.length()); } @@ -1411,7 +1411,7 @@ TYPE_ENTITY_LITERAL_ID KVstore::getIDByLiteral(string _literal) const { //this->load_trie(); - _literal = trie->Compress(_literal); +// _literal = trie->Compress(_literal); return this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length()); //TYPE_ENTITY_LITERAL_ID id = this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length()); //if(id != INVALID) @@ -1428,7 +1428,7 @@ KVstore::setIDByLiteral(string _literal, TYPE_ENTITY_LITERAL_ID _id) //this->load_trie(); int len = _literal.length(); char* str = new char[len]; - _literal = trie->Compress(_literal); +// _literal = trie->Compress(_literal); memcpy(str, _literal.c_str(), len); return this->addValueByKey(this->literal2id, str, len, _id); @@ -1493,7 +1493,7 @@ KVstore::getLiteralByID(TYPE_ENTITY_LITERAL_ID _id) const } string _ret = string(_tmp, _len); delete []_tmp; - _ret = trie->Uncompress(_ret, _ret.length()); +// _ret = trie->Uncompress(_ret, _ret.length()); return _ret; } diff --git a/KVstore/KVstore.h b/KVstore/KVstore.h index 79fc2b4..f464afd 100644 --- a/KVstore/KVstore.h +++ b/KVstore/KVstore.h @@ -12,7 +12,7 @@ #include "../Util/Util.h" #include "../Util/VList.h" #include "Tree.h" -#include "../Trie/Trie.h" +//#include "../Trie/Trie.h" #include "IVArray/IVArray.h" #include "ISArray/ISArray.h" @@ -169,14 +169,14 @@ public: //for so2p bool getpreIDlistBysubIDobjID(TYPE_ENTITY_LITERAL_ID _subID, TYPE_ENTITY_LITERAL_ID _objID, unsigned*& _preidlist, unsigned& _list_len, bool _no_duplicate = false) const; - bool load_trie(); + //bool load_trie(); private: std::string store_path; - std::string dictionary_store_path; +// std::string dictionary_store_path; - Trie *trie; + //Trie *trie; SITree* entity2id; //ISTree* id2entity; diff --git a/StringIndex/StringIndex.cpp b/StringIndex/StringIndex.cpp index 341d7ac..ca144f5 100644 --- a/StringIndex/StringIndex.cpp +++ b/StringIndex/StringIndex.cpp @@ -81,7 +81,7 @@ void StringIndexFile::load() this->empty_offset = max(this->empty_offset, (*this->index_table)[i].offset + (long)(*this->index_table)[i].length); } - trie->LoadTrie(dictionary_path); +// trie->LoadTrie(dictionary_path); } bool StringIndexFile::randomAccess(unsigned id, string *str, bool real) @@ -110,10 +110,10 @@ bool StringIndexFile::randomAccess(unsigned id, string *str, bool real) //cout<<"check: "<<*str<Uncompress(*str, str->length());//Uncompresss - } +// if (real) +// { +// *str = trie->Uncompress(*str, str->length());//Uncompresss +// } //if(id == 9) //{ //cout<<"check: "<<*str<buffer[length] = '\0'; *this->request[pos].str = string(this->buffer); - *this->request[pos].str = trie->Uncompress( - *this->request[pos].str, this->request[pos].str->length()); +// *this->request[pos].str = trie->Uncompress( +// *this->request[pos].str, this->request[pos].str->length()); pos++; } @@ -216,8 +216,8 @@ void StringIndexFile::trySequenceAccess(bool real) this->buffer[length] = '\0'; *this->request[pos].str += string(this->buffer); - *this->request[pos].str = trie->Uncompress( - *this->request[pos].str, this->request[pos].str->length()); +// *this->request[pos].str = trie->Uncompress( +// *this->request[pos].str, this->request[pos].str->length()); pos++; while (pos < (int)this->request.size() && this->request[pos - 1].offset == this->request[pos].offset) diff --git a/StringIndex/StringIndex.h b/StringIndex/StringIndex.h index 914486c..2a6364d 100644 --- a/StringIndex/StringIndex.h +++ b/StringIndex/StringIndex.h @@ -52,10 +52,10 @@ class StringIndexFile }; std::vector request; - std::string dictionary_path; +// std::string dictionary_path; public: - Trie *trie; +// Trie *trie; StringIndexFile(StringIndexFileType _type, std::string _dir, unsigned _num):type(_type), num(_num), empty_offset(0), index_file(NULL), value_file(NULL), buffer_size(0), buffer(NULL) { @@ -67,8 +67,8 @@ class StringIndexFile this->loc = _dir + "/predicate_"; this->index_table = new std::vector; - dictionary_path = _dir + "/../dictionary.dc"; - trie = new Trie; +// dictionary_path = _dir + "/../dictionary.dc"; +// trie = new Trie; } ~StringIndexFile() { @@ -79,8 +79,8 @@ class StringIndexFile if (this->buffer != NULL) delete[] this->buffer; delete this->index_table; - if (this->trie != NULL) - delete trie; +// if (this->trie != NULL) +// delete trie; } void clear() @@ -88,7 +88,7 @@ class StringIndexFile this->index_file = NULL; this->value_file = NULL; this->index_table = NULL; - this->trie = NULL; +// this->trie = NULL; } void setNum(unsigned _num); @@ -132,12 +132,12 @@ class StringIndex unsigned literal_buffer_size; public: - Trie *trie; +// Trie *trie; StringIndex(std::string _dir, unsigned _entity_num = 0, unsigned _literal_num = 0, unsigned _predicate_num = 0): entity(StringIndexFile::Entity, _dir, _entity_num), literal(StringIndexFile::Literal, _dir, _literal_num), predicate(StringIndexFile::Predicate, _dir, _predicate_num) { - trie = entity.trie; +// trie = entity.trie; } void clear() From 760e9ae9b9d77eb6e452443914c076e9671b1da5 Mon Sep 17 00:00:00 2001 From: Zongyue Date: Fri, 19 Oct 2018 05:26:01 +0800 Subject: [PATCH 2/3] Revert "Fix: remove trie and fix bugs in IS(IV)BlockManager.cpp" This reverts commit 15521f61320e52852267183dc544438aa332e233. --- Database/Database.cpp | 134 ++++++++++++++--------------- Database/Database.h | 2 +- KVstore/ISArray/ISBlockManager.cpp | 8 +- KVstore/IVArray/IVBlockManager.cpp | 4 +- KVstore/KVstore.cpp | 50 +++++------ KVstore/KVstore.h | 8 +- StringIndex/StringIndex.cpp | 18 ++-- StringIndex/StringIndex.h | 18 ++-- 8 files changed, 118 insertions(+), 124 deletions(-) diff --git a/Database/Database.cpp b/Database/Database.cpp index 7189b7b..ed8f27c 100644 --- a/Database/Database.cpp +++ b/Database/Database.cpp @@ -51,7 +51,7 @@ Database::Database() this->if_loaded = false; -// this->trie = NULL; + this->trie = NULL; //this->resetIDinfo(); this->initIDinfo(); @@ -103,7 +103,7 @@ Database::Database(string _name) this->query_cache = new QueryCache(); -// this->trie = NULL; + this->trie = NULL; //this->resetIDinfo(); this->initIDinfo(); @@ -685,8 +685,8 @@ Database::load() return false; } -// if(!(this->kvstore)->load_trie()) -// return false; + if(!(this->kvstore)->load_trie()) + return false; //NOTICE: we should also run some heavy work in the main thread this->stringindex->load(); @@ -736,17 +736,17 @@ Database::load() //this->warmUp(); //DEBUG:the warmUp() calls query(), which will also output results, this is not we want - // Load trie (trie has been removed) + // Load trie -// if (trie != NULL) -// delete trie; -// trie = new Trie; + if (trie != NULL) + delete trie; + trie = new Trie; -// string dictionary_path = store_path + "/dictionary.dc"; -// if (!trie->LoadTrie(dictionary_path)) -// { -// return false; -// } + string dictionary_path = store_path + "/dictionary.dc"; + if (!trie->LoadTrie(dictionary_path)) + { + return false; + } this->if_loaded = true; @@ -1259,11 +1259,11 @@ Database::unload() this->if_loaded = false; this->clear_update_log(); -// if (this->trie != NULL) -// { -// delete this->trie; -// trie = NULL; -// } + if (this->trie != NULL) + { + delete this->trie; + trie = NULL; + } return true; } @@ -1395,7 +1395,7 @@ Database::get_query_parse_lock() int Database::query(const string _query, ResultSet& _result_set, FILE* _fp) { -// string dictionary_store_path = this->store_path + "/dictionary.dc"; + string dictionary_store_path = this->store_path + "/dictionary.dc"; GeneralEvaluation general_evaluation(this->vstree, this->kvstore, this->stringindex, this->query_cache, this->pre2num, this->limitID_predicate, this->limitID_literal,this->limitID_entity); @@ -1444,16 +1444,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) long tv_bfget = Util::get_cur_time(); //NOTICE: this lock lock ensures that StringIndex is visited sequentially this->getFinalResult_lock.lock(); -// if (trie == NULL) -// { -// trie = new Trie; -// string dictionary_path = this->store_path + "/dictionary.dc"; -// if (!trie->LoadTrie(dictionary_path)) -// { -// exit(0); -// } -// trie->LoadDictionary(); -// } + if (trie == NULL) + { + trie = new Trie; + string dictionary_path = this->store_path + "/dictionary.dc"; + if (!trie->LoadTrie(dictionary_path)) + { + exit(0); + } + trie->LoadDictionary(); + } general_evaluation.getFinalResult(_result_set); this->getFinalResult_lock.unlock(); long tv_afget = Util::get_cur_time(); @@ -1485,16 +1485,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) TripleWithObjType *update_triple = NULL; TYPE_TRIPLE_NUM update_triple_num = 0; -// if (trie == NULL) -// { -// trie = new Trie; -// string dictionary_path = this->store_path + "/dictionary.dc"; -// if (!trie->LoadTrie(dictionary_path)) -// { -// exit(0); -// } -// trie->LoadDictionary(); -// } + if (trie == NULL) + { + trie = new Trie; + string dictionary_path = this->store_path + "/dictionary.dc"; + if (!trie->LoadTrie(dictionary_path)) + { + exit(0); + } + trie->LoadDictionary(); + } if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Insert_Data || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Data) @@ -1519,7 +1519,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) update_pattern.sub_group_pattern[i].pattern.object.value, object_type); // Compress -// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); + update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); } else { @@ -1547,7 +1547,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getDeletePatterns(), update_triple, update_triple_num); for(int i = 0; i < update_triple_num; i++) { -// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); + update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); } success_num = remove(update_triple, update_triple_num); } @@ -1556,7 +1556,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getInsertPatterns(), update_triple, update_triple_num); for(int i = 0; i < update_triple_num; i++) { -// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); + update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); } success_num = insert(update_triple, update_triple_num); } @@ -2413,7 +2413,7 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file) (this->kvstore)->open_id2predicate(KVstore::CREATE_MODE); (this->kvstore)->open_literal2id(KVstore::CREATE_MODE); (this->kvstore)->open_id2literal(KVstore::CREATE_MODE); -// (this->kvstore)->load_trie(); + (this->kvstore)->load_trie(); } //Util::logging("finish initial sub2id_pre2id_obj2id"); @@ -2457,19 +2457,19 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file) RDFParser _parser(_fin); // Initialize trie -// string dictionary_store_path = this->store_path + "/dictionary.dc"; -// this->trie = new Trie (_rdf_file, dictionary_store_path); -// if (!trie->isInitialized()) -// { -// cout << "Fail to initialize trie" << endl; -// exit(0); -// } + string dictionary_store_path = this->store_path + "/dictionary.dc"; + this->trie = new Trie (_rdf_file, dictionary_store_path); + if (!trie->isInitialized()) + { + cout << "Fail to initialize trie" << endl; + exit(0); + } -// if(!trie->WriteDown()) -// { -// cout << "Fail to write down dictionary" << endl; -// exit(0); -// } + if(!trie->WriteDown()) + { + cout << "Fail to write down dictionary" << endl; + exit(0); + } //Util::logging("==> while(true)"); while (true) @@ -2515,8 +2515,8 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file) //We need a thread pool! // Compress triple begin -// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE); -// triple_array[i] = compressed_triple; + TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE); + triple_array[i] = compressed_triple; //Compress triple end // For subject @@ -3036,11 +3036,11 @@ Database::insert(std::string _rdf_file, bool _is_restore) } //Compress triple here -// for(int i = 0; i < parse_triple_num; i++) -// { -// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); -// triple_array[i] = compressed_triple; -// } + for(int i = 0; i < parse_triple_num; i++) + { + TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); + triple_array[i] = compressed_triple; + } //Process the Triple one by one success_num += this->insert(triple_array, parse_triple_num, _is_restore); @@ -3136,11 +3136,11 @@ Database::remove(std::string _rdf_file, bool _is_restore) } //Compress triple -// for(int i = 0; i < parse_triple_num; i++) -// { -// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); -// triple_array[i] = compressed_triple; -// } + for(int i = 0; i < parse_triple_num; i++) + { + TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); + triple_array[i] = compressed_triple; + } success_num += this->remove(triple_array, parse_triple_num, _is_restore); diff --git a/Database/Database.h b/Database/Database.h index 61ba013..cca19d1 100644 --- a/Database/Database.h +++ b/Database/Database.h @@ -155,7 +155,7 @@ private: QueryCache *query_cache; - //Trie *trie; + Trie *trie; void setStringBuffer(); void warmUp(); diff --git a/KVstore/ISArray/ISBlockManager.cpp b/KVstore/ISArray/ISBlockManager.cpp index 302e73b..6ac9399 100644 --- a/KVstore/ISArray/ISBlockManager.cpp +++ b/KVstore/ISArray/ISBlockManager.cpp @@ -335,10 +335,8 @@ ISBlockManager::FreeBlocks(const unsigned index) { cur_index = it->first; curlen += it->second; - auto tmp_key = it->second, tmp_val = it->first; index_len_map.erase(it); - //len_index_map.erase(make_pair(it->second,it->first)); - len_index_map.erase(make_pair(tmp_key, tmp_val)); + len_index_map.erase(make_pair(it->second,it->first)); } } @@ -348,10 +346,8 @@ ISBlockManager::FreeBlocks(const unsigned index) if (curlen + cur_index == it->first) // block after is free { curlen += it->second; - auto tmp_key = it->second, tmp_val = it->first; index_len_map.erase(it); - //len_index_map.erase(make_pair(it->second,it->first)); - len_index_map.erase(make_pair(tmp_key, tmp_val)); + len_index_map.erase(make_pair(it->second,it->first)); } } diff --git a/KVstore/IVArray/IVBlockManager.cpp b/KVstore/IVArray/IVBlockManager.cpp index 2cebe5b..212e4ec 100644 --- a/KVstore/IVArray/IVBlockManager.cpp +++ b/KVstore/IVArray/IVBlockManager.cpp @@ -335,10 +335,8 @@ IVBlockManager::FreeBlocks(const unsigned index) { cur_index = it->first; curlen += it->second; - auto tmp_key = it->second, tmp_val = it->first; index_len_map.erase(it); - //len_index_map.erase(make_pair(it->second,it->first)); - len_index_map.erase(make_pair(tmp_key, tmp_val)); + len_index_map.erase(make_pair(it->second,it->first)); } } diff --git a/KVstore/KVstore.cpp b/KVstore/KVstore.cpp index 7343ec7..b9db2b3 100644 --- a/KVstore/KVstore.cpp +++ b/KVstore/KVstore.cpp @@ -16,8 +16,8 @@ KVstore::KVstore(string _store_path) { this->store_path = _store_path; -// this->dictionary_store_path = _store_path + "/../dictionary.dc"; - //this->trie = NULL; + this->dictionary_store_path = _store_path + "/../dictionary.dc"; + this->trie = NULL; this->entity2id = NULL; this->id2entity = NULL; @@ -97,9 +97,9 @@ KVstore::release() delete this->objID2values_literal; this->objID2values_literal = NULL; - //if (trie != NULL) - // delete this->trie; - //this->trie = NULL; + if (trie != NULL) + delete this->trie; + this->trie = NULL; } void @@ -120,18 +120,18 @@ KVstore::open() this->open_objID2values(KVstore::READ_WRITE_MODE); this->open_preID2values(KVstore::READ_WRITE_MODE); - //this->trie = new Trie; - //trie->LoadTrie(dictionary_store_path); + this->trie = new Trie; + trie->LoadTrie(dictionary_store_path); } -//bool -//KVstore::load_trie() -//{ -// if (trie != NULL) return true; +bool +KVstore::load_trie() +{ + if (trie != NULL) return true; -// trie = new Trie; -// return trie->LoadTrie(dictionary_store_path); -//} + trie = new Trie; + return trie->LoadTrie(dictionary_store_path); +} string KVstore::getStringByID(TYPE_ENTITY_LITERAL_ID _id) @@ -1099,7 +1099,7 @@ KVstore::subIDByEntity(string _entity) //_entity will not be released befor ethis function ends //so _entity.c_str() is a valid const char* //this->load_trie(); - //_entity = trie->Compress(_entity); + _entity = trie->Compress(_entity); //return this->entity2id->remove(_entity.c_str(), _entity.length()); return this->removeKey(this->entity2id, _entity.c_str(), _entity.length()); } @@ -1108,7 +1108,7 @@ TYPE_ENTITY_LITERAL_ID KVstore::getIDByEntity(string _entity) const { //this->load_trie(); - //_entity = trie->Compress(_entity); + _entity = trie->Compress(_entity); return this->getIDByStr(this->entity2id, _entity.c_str(), _entity.length()); } @@ -1121,7 +1121,7 @@ KVstore::setIDByEntity(string _entity, TYPE_ENTITY_LITERAL_ID _id) char* str = new char[len]; //this->load_trie(); - //_entity = trie->Compress(_entity); + _entity = trie->Compress(_entity); memcpy(str, _entity.c_str(), len); return this->addValueByKey(this->entity2id, str, len, _id); } @@ -1196,7 +1196,7 @@ KVstore::getEntityByID(TYPE_ENTITY_LITERAL_ID _id) const //cout << _tmp << endl; string _ret = string(_tmp, _len); -// _ret = trie->Uncompress(_ret, _ret.length()); + _ret = trie->Uncompress(_ret, _ret.length()); delete []_tmp; // cout << "string is: " << _ret << endl; // string test1 = _ret; @@ -1260,7 +1260,7 @@ bool KVstore::subIDByPredicate(string _predicate) { //this->load_trie(); -// _predicate = trie->Compress(_predicate); + _predicate = trie->Compress(_predicate); //return this->predicate2id->remove(_predicate.c_str(), _predicate.length()); return this->removeKey(this->predicate2id, _predicate.c_str(), _predicate.length()); } @@ -1269,7 +1269,7 @@ TYPE_PREDICATE_ID KVstore::getIDByPredicate(string _predicate) const { //this->load_trie(); -// _predicate = trie->Compress(_predicate); + _predicate = trie->Compress(_predicate); return this->getIDByStr(this->predicate2id, _predicate.c_str(), _predicate.length()); } @@ -1282,7 +1282,7 @@ KVstore::setIDByPredicate(string _predicate, TYPE_PREDICATE_ID _id) char* str = new char[len]; //this->load_trie(); -// _predicate = trie->Compress(_predicate); + _predicate = trie->Compress(_predicate); memcpy(str, _predicate.c_str(), len); return this->addValueByKey(this->predicate2id, str, len, _id); } @@ -1402,7 +1402,7 @@ bool KVstore::subIDByLiteral(string _literal) { //this->load_trie(); -// _literal = trie->Compress(_literal); + _literal = trie->Compress(_literal); //return this->literal2id->remove(_literal.c_str(), _literal.length()); return this->removeKey(this->literal2id, _literal.c_str(), _literal.length()); } @@ -1411,7 +1411,7 @@ TYPE_ENTITY_LITERAL_ID KVstore::getIDByLiteral(string _literal) const { //this->load_trie(); -// _literal = trie->Compress(_literal); + _literal = trie->Compress(_literal); return this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length()); //TYPE_ENTITY_LITERAL_ID id = this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length()); //if(id != INVALID) @@ -1428,7 +1428,7 @@ KVstore::setIDByLiteral(string _literal, TYPE_ENTITY_LITERAL_ID _id) //this->load_trie(); int len = _literal.length(); char* str = new char[len]; -// _literal = trie->Compress(_literal); + _literal = trie->Compress(_literal); memcpy(str, _literal.c_str(), len); return this->addValueByKey(this->literal2id, str, len, _id); @@ -1493,7 +1493,7 @@ KVstore::getLiteralByID(TYPE_ENTITY_LITERAL_ID _id) const } string _ret = string(_tmp, _len); delete []_tmp; -// _ret = trie->Uncompress(_ret, _ret.length()); + _ret = trie->Uncompress(_ret, _ret.length()); return _ret; } diff --git a/KVstore/KVstore.h b/KVstore/KVstore.h index f464afd..79fc2b4 100644 --- a/KVstore/KVstore.h +++ b/KVstore/KVstore.h @@ -12,7 +12,7 @@ #include "../Util/Util.h" #include "../Util/VList.h" #include "Tree.h" -//#include "../Trie/Trie.h" +#include "../Trie/Trie.h" #include "IVArray/IVArray.h" #include "ISArray/ISArray.h" @@ -169,14 +169,14 @@ public: //for so2p bool getpreIDlistBysubIDobjID(TYPE_ENTITY_LITERAL_ID _subID, TYPE_ENTITY_LITERAL_ID _objID, unsigned*& _preidlist, unsigned& _list_len, bool _no_duplicate = false) const; - //bool load_trie(); + bool load_trie(); private: std::string store_path; -// std::string dictionary_store_path; + std::string dictionary_store_path; - //Trie *trie; + Trie *trie; SITree* entity2id; //ISTree* id2entity; diff --git a/StringIndex/StringIndex.cpp b/StringIndex/StringIndex.cpp index ca144f5..341d7ac 100644 --- a/StringIndex/StringIndex.cpp +++ b/StringIndex/StringIndex.cpp @@ -81,7 +81,7 @@ void StringIndexFile::load() this->empty_offset = max(this->empty_offset, (*this->index_table)[i].offset + (long)(*this->index_table)[i].length); } -// trie->LoadTrie(dictionary_path); + trie->LoadTrie(dictionary_path); } bool StringIndexFile::randomAccess(unsigned id, string *str, bool real) @@ -110,10 +110,10 @@ bool StringIndexFile::randomAccess(unsigned id, string *str, bool real) //cout<<"check: "<<*str<Uncompress(*str, str->length());//Uncompresss -// } + if (real) + { + *str = trie->Uncompress(*str, str->length());//Uncompresss + } //if(id == 9) //{ //cout<<"check: "<<*str<buffer[length] = '\0'; *this->request[pos].str = string(this->buffer); -// *this->request[pos].str = trie->Uncompress( -// *this->request[pos].str, this->request[pos].str->length()); + *this->request[pos].str = trie->Uncompress( + *this->request[pos].str, this->request[pos].str->length()); pos++; } @@ -216,8 +216,8 @@ void StringIndexFile::trySequenceAccess(bool real) this->buffer[length] = '\0'; *this->request[pos].str += string(this->buffer); -// *this->request[pos].str = trie->Uncompress( -// *this->request[pos].str, this->request[pos].str->length()); + *this->request[pos].str = trie->Uncompress( + *this->request[pos].str, this->request[pos].str->length()); pos++; while (pos < (int)this->request.size() && this->request[pos - 1].offset == this->request[pos].offset) diff --git a/StringIndex/StringIndex.h b/StringIndex/StringIndex.h index 2a6364d..914486c 100644 --- a/StringIndex/StringIndex.h +++ b/StringIndex/StringIndex.h @@ -52,10 +52,10 @@ class StringIndexFile }; std::vector request; -// std::string dictionary_path; + std::string dictionary_path; public: -// Trie *trie; + Trie *trie; StringIndexFile(StringIndexFileType _type, std::string _dir, unsigned _num):type(_type), num(_num), empty_offset(0), index_file(NULL), value_file(NULL), buffer_size(0), buffer(NULL) { @@ -67,8 +67,8 @@ class StringIndexFile this->loc = _dir + "/predicate_"; this->index_table = new std::vector; -// dictionary_path = _dir + "/../dictionary.dc"; -// trie = new Trie; + dictionary_path = _dir + "/../dictionary.dc"; + trie = new Trie; } ~StringIndexFile() { @@ -79,8 +79,8 @@ class StringIndexFile if (this->buffer != NULL) delete[] this->buffer; delete this->index_table; -// if (this->trie != NULL) -// delete trie; + if (this->trie != NULL) + delete trie; } void clear() @@ -88,7 +88,7 @@ class StringIndexFile this->index_file = NULL; this->value_file = NULL; this->index_table = NULL; -// this->trie = NULL; + this->trie = NULL; } void setNum(unsigned _num); @@ -132,12 +132,12 @@ class StringIndex unsigned literal_buffer_size; public: -// Trie *trie; + Trie *trie; StringIndex(std::string _dir, unsigned _entity_num = 0, unsigned _literal_num = 0, unsigned _predicate_num = 0): entity(StringIndexFile::Entity, _dir, _entity_num), literal(StringIndexFile::Literal, _dir, _literal_num), predicate(StringIndexFile::Predicate, _dir, _predicate_num) { -// trie = entity.trie; + trie = entity.trie; } void clear() From 0f0eb86ba90c1094e5ae78a34abd950505873607 Mon Sep 17 00:00:00 2001 From: Zongyue Date: Fri, 19 Oct 2018 05:37:22 +0800 Subject: [PATCH 3/3] Fix: remove trie, fix bugs in IS(IV)BlockManager, remove curKeyNum --- Database/Database.cpp | 134 ++++++++++++++--------------- Database/Database.h | 2 +- KVstore/ISArray/ISArray.cpp | 4 - KVstore/ISArray/ISArray.h | 1 - KVstore/ISArray/ISBlockManager.cpp | 8 +- KVstore/IVArray/IVArray.cpp | 4 - KVstore/IVArray/IVArray.h | 1 - KVstore/IVArray/IVBlockManager.cpp | 4 +- KVstore/KVstore.cpp | 50 +++++------ KVstore/KVstore.h | 8 +- StringIndex/StringIndex.cpp | 18 ++-- StringIndex/StringIndex.h | 18 ++-- 12 files changed, 124 insertions(+), 128 deletions(-) diff --git a/Database/Database.cpp b/Database/Database.cpp index ed8f27c..7189b7b 100644 --- a/Database/Database.cpp +++ b/Database/Database.cpp @@ -51,7 +51,7 @@ Database::Database() this->if_loaded = false; - this->trie = NULL; +// this->trie = NULL; //this->resetIDinfo(); this->initIDinfo(); @@ -103,7 +103,7 @@ Database::Database(string _name) this->query_cache = new QueryCache(); - this->trie = NULL; +// this->trie = NULL; //this->resetIDinfo(); this->initIDinfo(); @@ -685,8 +685,8 @@ Database::load() return false; } - if(!(this->kvstore)->load_trie()) - return false; +// if(!(this->kvstore)->load_trie()) +// return false; //NOTICE: we should also run some heavy work in the main thread this->stringindex->load(); @@ -736,17 +736,17 @@ Database::load() //this->warmUp(); //DEBUG:the warmUp() calls query(), which will also output results, this is not we want - // Load trie + // Load trie (trie has been removed) - if (trie != NULL) - delete trie; - trie = new Trie; +// if (trie != NULL) +// delete trie; +// trie = new Trie; - string dictionary_path = store_path + "/dictionary.dc"; - if (!trie->LoadTrie(dictionary_path)) - { - return false; - } +// string dictionary_path = store_path + "/dictionary.dc"; +// if (!trie->LoadTrie(dictionary_path)) +// { +// return false; +// } this->if_loaded = true; @@ -1259,11 +1259,11 @@ Database::unload() this->if_loaded = false; this->clear_update_log(); - if (this->trie != NULL) - { - delete this->trie; - trie = NULL; - } +// if (this->trie != NULL) +// { +// delete this->trie; +// trie = NULL; +// } return true; } @@ -1395,7 +1395,7 @@ Database::get_query_parse_lock() int Database::query(const string _query, ResultSet& _result_set, FILE* _fp) { - string dictionary_store_path = this->store_path + "/dictionary.dc"; +// string dictionary_store_path = this->store_path + "/dictionary.dc"; GeneralEvaluation general_evaluation(this->vstree, this->kvstore, this->stringindex, this->query_cache, this->pre2num, this->limitID_predicate, this->limitID_literal,this->limitID_entity); @@ -1444,16 +1444,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) long tv_bfget = Util::get_cur_time(); //NOTICE: this lock lock ensures that StringIndex is visited sequentially this->getFinalResult_lock.lock(); - if (trie == NULL) - { - trie = new Trie; - string dictionary_path = this->store_path + "/dictionary.dc"; - if (!trie->LoadTrie(dictionary_path)) - { - exit(0); - } - trie->LoadDictionary(); - } +// if (trie == NULL) +// { +// trie = new Trie; +// string dictionary_path = this->store_path + "/dictionary.dc"; +// if (!trie->LoadTrie(dictionary_path)) +// { +// exit(0); +// } +// trie->LoadDictionary(); +// } general_evaluation.getFinalResult(_result_set); this->getFinalResult_lock.unlock(); long tv_afget = Util::get_cur_time(); @@ -1485,16 +1485,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) TripleWithObjType *update_triple = NULL; TYPE_TRIPLE_NUM update_triple_num = 0; - if (trie == NULL) - { - trie = new Trie; - string dictionary_path = this->store_path + "/dictionary.dc"; - if (!trie->LoadTrie(dictionary_path)) - { - exit(0); - } - trie->LoadDictionary(); - } +// if (trie == NULL) +// { +// trie = new Trie; +// string dictionary_path = this->store_path + "/dictionary.dc"; +// if (!trie->LoadTrie(dictionary_path)) +// { +// exit(0); +// } +// trie->LoadDictionary(); +// } if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Insert_Data || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Data) @@ -1519,7 +1519,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) update_pattern.sub_group_pattern[i].pattern.object.value, object_type); // Compress - update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); +// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); } else { @@ -1547,7 +1547,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getDeletePatterns(), update_triple, update_triple_num); for(int i = 0; i < update_triple_num; i++) { - update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); +// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); } success_num = remove(update_triple, update_triple_num); } @@ -1556,7 +1556,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp) general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getInsertPatterns(), update_triple, update_triple_num); for(int i = 0; i < update_triple_num; i++) { - update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); +// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE); } success_num = insert(update_triple, update_triple_num); } @@ -2413,7 +2413,7 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file) (this->kvstore)->open_id2predicate(KVstore::CREATE_MODE); (this->kvstore)->open_literal2id(KVstore::CREATE_MODE); (this->kvstore)->open_id2literal(KVstore::CREATE_MODE); - (this->kvstore)->load_trie(); +// (this->kvstore)->load_trie(); } //Util::logging("finish initial sub2id_pre2id_obj2id"); @@ -2457,19 +2457,19 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file) RDFParser _parser(_fin); // Initialize trie - string dictionary_store_path = this->store_path + "/dictionary.dc"; - this->trie = new Trie (_rdf_file, dictionary_store_path); - if (!trie->isInitialized()) - { - cout << "Fail to initialize trie" << endl; - exit(0); - } +// string dictionary_store_path = this->store_path + "/dictionary.dc"; +// this->trie = new Trie (_rdf_file, dictionary_store_path); +// if (!trie->isInitialized()) +// { +// cout << "Fail to initialize trie" << endl; +// exit(0); +// } - if(!trie->WriteDown()) - { - cout << "Fail to write down dictionary" << endl; - exit(0); - } +// if(!trie->WriteDown()) +// { +// cout << "Fail to write down dictionary" << endl; +// exit(0); +// } //Util::logging("==> while(true)"); while (true) @@ -2515,8 +2515,8 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file) //We need a thread pool! // Compress triple begin - TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE); - triple_array[i] = compressed_triple; +// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE); +// triple_array[i] = compressed_triple; //Compress triple end // For subject @@ -3036,11 +3036,11 @@ Database::insert(std::string _rdf_file, bool _is_restore) } //Compress triple here - for(int i = 0; i < parse_triple_num; i++) - { - TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); - triple_array[i] = compressed_triple; - } +// for(int i = 0; i < parse_triple_num; i++) +// { +// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); +// triple_array[i] = compressed_triple; +// } //Process the Triple one by one success_num += this->insert(triple_array, parse_triple_num, _is_restore); @@ -3136,11 +3136,11 @@ Database::remove(std::string _rdf_file, bool _is_restore) } //Compress triple - for(int i = 0; i < parse_triple_num; i++) - { - TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); - triple_array[i] = compressed_triple; - } +// for(int i = 0; i < parse_triple_num; i++) +// { +// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE); +// triple_array[i] = compressed_triple; +// } success_num += this->remove(triple_array, parse_triple_num, _is_restore); diff --git a/Database/Database.h b/Database/Database.h index cca19d1..61ba013 100644 --- a/Database/Database.h +++ b/Database/Database.h @@ -155,7 +155,7 @@ private: QueryCache *query_cache; - Trie *trie; + //Trie *trie; void setStringBuffer(); void warmUp(); diff --git a/KVstore/ISArray/ISArray.cpp b/KVstore/ISArray/ISArray.cpp index 2f8ea7e..a369fe1 100644 --- a/KVstore/ISArray/ISArray.cpp +++ b/KVstore/ISArray/ISArray.cpp @@ -16,7 +16,6 @@ ISArray::ISArray() dir_path = ""; ISfile_name = ""; BM = NULL; - CurKeyNum = 0; CurEntryNum = 0; CurCacheSize = 0; CurEntryNumChange = false; @@ -55,7 +54,6 @@ ISArray::ISArray(string _dir_path, string _filename, string mode, unsigned long // temp is the smallest number >= _key_num and mod SET_KEY_INC = 0 unsigned temp = ((_key_num + (1 << 10) - 1) >> 10) << 10; - CurKeyNum = 0; CurEntryNum = max(temp, SETKEYNUM); CurEntryNumChange = true; @@ -341,7 +339,6 @@ ISArray::insert(unsigned _key, char *_str, unsigned _len) } bool Alloc = false; - CurKeyNum++; //if (CurKeyNum >= CurEntryNum) // need to realloc if (_key >= CurEntryNum) { @@ -391,7 +388,6 @@ ISArray::remove(unsigned _key) return false; } - CurKeyNum--; unsigned store = array[_key].getStore(); BM->FreeBlocks(store); diff --git a/KVstore/ISArray/ISArray.h b/KVstore/ISArray/ISArray.h index 4ce7549..502a51a 100644 --- a/KVstore/ISArray/ISArray.h +++ b/KVstore/ISArray/ISArray.h @@ -35,7 +35,6 @@ private: string dir_path; ISBlockManager *BM; unsigned CurEntryNum; // how many entries are available - unsigned CurKeyNum; // how many keys are stored bool CurEntryNumChange; //Cache diff --git a/KVstore/ISArray/ISBlockManager.cpp b/KVstore/ISArray/ISBlockManager.cpp index 6ac9399..302e73b 100644 --- a/KVstore/ISArray/ISBlockManager.cpp +++ b/KVstore/ISArray/ISBlockManager.cpp @@ -335,8 +335,10 @@ ISBlockManager::FreeBlocks(const unsigned index) { cur_index = it->first; curlen += it->second; + auto tmp_key = it->second, tmp_val = it->first; index_len_map.erase(it); - len_index_map.erase(make_pair(it->second,it->first)); + //len_index_map.erase(make_pair(it->second,it->first)); + len_index_map.erase(make_pair(tmp_key, tmp_val)); } } @@ -346,8 +348,10 @@ ISBlockManager::FreeBlocks(const unsigned index) if (curlen + cur_index == it->first) // block after is free { curlen += it->second; + auto tmp_key = it->second, tmp_val = it->first; index_len_map.erase(it); - len_index_map.erase(make_pair(it->second,it->first)); + //len_index_map.erase(make_pair(it->second,it->first)); + len_index_map.erase(make_pair(tmp_key, tmp_val)); } } diff --git a/KVstore/IVArray/IVArray.cpp b/KVstore/IVArray/IVArray.cpp index b2c919f..7067ca5 100644 --- a/KVstore/IVArray/IVArray.cpp +++ b/KVstore/IVArray/IVArray.cpp @@ -16,7 +16,6 @@ IVArray::IVArray() dir_path = ""; IVfile_name = ""; BM = NULL; - CurKeyNum = 0; CurEntryNum = 0; CurCacheSize = 0; CurEntryNumChange = false; @@ -63,7 +62,6 @@ IVArray::IVArray(string _dir_path, string _filename, string mode, unsigned long // temp is the smallest number >= _key_num and mod SET_KEY_INC = 0 unsigned temp = ((_key_num + (1 << 10) - 1) >> 10) << 10; - CurKeyNum = 0; CurEntryNum = max(temp, SETKEYNUM); CurEntryNumChange = true; @@ -346,7 +344,6 @@ IVArray::insert(unsigned _key, char *_str, unsigned _len) return false; } - CurKeyNum++; //if (CurKeyNum >= CurEntryNum) // need to realloc if (_key >= CurEntryNum) { @@ -402,7 +399,6 @@ IVArray::remove(unsigned _key) return false; } - CurKeyNum--; unsigned store = array[_key].getStore(); BM->FreeBlocks(store); diff --git a/KVstore/IVArray/IVArray.h b/KVstore/IVArray/IVArray.h index 8932183..2cfe636 100644 --- a/KVstore/IVArray/IVArray.h +++ b/KVstore/IVArray/IVArray.h @@ -36,7 +36,6 @@ private: string dir_path; IVBlockManager *BM; unsigned CurEntryNum; // how many entries are available - unsigned CurKeyNum; // how many keys are stored bool CurEntryNumChange; //Cache diff --git a/KVstore/IVArray/IVBlockManager.cpp b/KVstore/IVArray/IVBlockManager.cpp index 212e4ec..2cebe5b 100644 --- a/KVstore/IVArray/IVBlockManager.cpp +++ b/KVstore/IVArray/IVBlockManager.cpp @@ -335,8 +335,10 @@ IVBlockManager::FreeBlocks(const unsigned index) { cur_index = it->first; curlen += it->second; + auto tmp_key = it->second, tmp_val = it->first; index_len_map.erase(it); - len_index_map.erase(make_pair(it->second,it->first)); + //len_index_map.erase(make_pair(it->second,it->first)); + len_index_map.erase(make_pair(tmp_key, tmp_val)); } } diff --git a/KVstore/KVstore.cpp b/KVstore/KVstore.cpp index b9db2b3..7343ec7 100644 --- a/KVstore/KVstore.cpp +++ b/KVstore/KVstore.cpp @@ -16,8 +16,8 @@ KVstore::KVstore(string _store_path) { this->store_path = _store_path; - this->dictionary_store_path = _store_path + "/../dictionary.dc"; - this->trie = NULL; +// this->dictionary_store_path = _store_path + "/../dictionary.dc"; + //this->trie = NULL; this->entity2id = NULL; this->id2entity = NULL; @@ -97,9 +97,9 @@ KVstore::release() delete this->objID2values_literal; this->objID2values_literal = NULL; - if (trie != NULL) - delete this->trie; - this->trie = NULL; + //if (trie != NULL) + // delete this->trie; + //this->trie = NULL; } void @@ -120,18 +120,18 @@ KVstore::open() this->open_objID2values(KVstore::READ_WRITE_MODE); this->open_preID2values(KVstore::READ_WRITE_MODE); - this->trie = new Trie; - trie->LoadTrie(dictionary_store_path); + //this->trie = new Trie; + //trie->LoadTrie(dictionary_store_path); } -bool -KVstore::load_trie() -{ - if (trie != NULL) return true; +//bool +//KVstore::load_trie() +//{ +// if (trie != NULL) return true; - trie = new Trie; - return trie->LoadTrie(dictionary_store_path); -} +// trie = new Trie; +// return trie->LoadTrie(dictionary_store_path); +//} string KVstore::getStringByID(TYPE_ENTITY_LITERAL_ID _id) @@ -1099,7 +1099,7 @@ KVstore::subIDByEntity(string _entity) //_entity will not be released befor ethis function ends //so _entity.c_str() is a valid const char* //this->load_trie(); - _entity = trie->Compress(_entity); + //_entity = trie->Compress(_entity); //return this->entity2id->remove(_entity.c_str(), _entity.length()); return this->removeKey(this->entity2id, _entity.c_str(), _entity.length()); } @@ -1108,7 +1108,7 @@ TYPE_ENTITY_LITERAL_ID KVstore::getIDByEntity(string _entity) const { //this->load_trie(); - _entity = trie->Compress(_entity); + //_entity = trie->Compress(_entity); return this->getIDByStr(this->entity2id, _entity.c_str(), _entity.length()); } @@ -1121,7 +1121,7 @@ KVstore::setIDByEntity(string _entity, TYPE_ENTITY_LITERAL_ID _id) char* str = new char[len]; //this->load_trie(); - _entity = trie->Compress(_entity); + //_entity = trie->Compress(_entity); memcpy(str, _entity.c_str(), len); return this->addValueByKey(this->entity2id, str, len, _id); } @@ -1196,7 +1196,7 @@ KVstore::getEntityByID(TYPE_ENTITY_LITERAL_ID _id) const //cout << _tmp << endl; string _ret = string(_tmp, _len); - _ret = trie->Uncompress(_ret, _ret.length()); +// _ret = trie->Uncompress(_ret, _ret.length()); delete []_tmp; // cout << "string is: " << _ret << endl; // string test1 = _ret; @@ -1260,7 +1260,7 @@ bool KVstore::subIDByPredicate(string _predicate) { //this->load_trie(); - _predicate = trie->Compress(_predicate); +// _predicate = trie->Compress(_predicate); //return this->predicate2id->remove(_predicate.c_str(), _predicate.length()); return this->removeKey(this->predicate2id, _predicate.c_str(), _predicate.length()); } @@ -1269,7 +1269,7 @@ TYPE_PREDICATE_ID KVstore::getIDByPredicate(string _predicate) const { //this->load_trie(); - _predicate = trie->Compress(_predicate); +// _predicate = trie->Compress(_predicate); return this->getIDByStr(this->predicate2id, _predicate.c_str(), _predicate.length()); } @@ -1282,7 +1282,7 @@ KVstore::setIDByPredicate(string _predicate, TYPE_PREDICATE_ID _id) char* str = new char[len]; //this->load_trie(); - _predicate = trie->Compress(_predicate); +// _predicate = trie->Compress(_predicate); memcpy(str, _predicate.c_str(), len); return this->addValueByKey(this->predicate2id, str, len, _id); } @@ -1402,7 +1402,7 @@ bool KVstore::subIDByLiteral(string _literal) { //this->load_trie(); - _literal = trie->Compress(_literal); +// _literal = trie->Compress(_literal); //return this->literal2id->remove(_literal.c_str(), _literal.length()); return this->removeKey(this->literal2id, _literal.c_str(), _literal.length()); } @@ -1411,7 +1411,7 @@ TYPE_ENTITY_LITERAL_ID KVstore::getIDByLiteral(string _literal) const { //this->load_trie(); - _literal = trie->Compress(_literal); +// _literal = trie->Compress(_literal); return this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length()); //TYPE_ENTITY_LITERAL_ID id = this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length()); //if(id != INVALID) @@ -1428,7 +1428,7 @@ KVstore::setIDByLiteral(string _literal, TYPE_ENTITY_LITERAL_ID _id) //this->load_trie(); int len = _literal.length(); char* str = new char[len]; - _literal = trie->Compress(_literal); +// _literal = trie->Compress(_literal); memcpy(str, _literal.c_str(), len); return this->addValueByKey(this->literal2id, str, len, _id); @@ -1493,7 +1493,7 @@ KVstore::getLiteralByID(TYPE_ENTITY_LITERAL_ID _id) const } string _ret = string(_tmp, _len); delete []_tmp; - _ret = trie->Uncompress(_ret, _ret.length()); +// _ret = trie->Uncompress(_ret, _ret.length()); return _ret; } diff --git a/KVstore/KVstore.h b/KVstore/KVstore.h index 79fc2b4..f464afd 100644 --- a/KVstore/KVstore.h +++ b/KVstore/KVstore.h @@ -12,7 +12,7 @@ #include "../Util/Util.h" #include "../Util/VList.h" #include "Tree.h" -#include "../Trie/Trie.h" +//#include "../Trie/Trie.h" #include "IVArray/IVArray.h" #include "ISArray/ISArray.h" @@ -169,14 +169,14 @@ public: //for so2p bool getpreIDlistBysubIDobjID(TYPE_ENTITY_LITERAL_ID _subID, TYPE_ENTITY_LITERAL_ID _objID, unsigned*& _preidlist, unsigned& _list_len, bool _no_duplicate = false) const; - bool load_trie(); + //bool load_trie(); private: std::string store_path; - std::string dictionary_store_path; +// std::string dictionary_store_path; - Trie *trie; + //Trie *trie; SITree* entity2id; //ISTree* id2entity; diff --git a/StringIndex/StringIndex.cpp b/StringIndex/StringIndex.cpp index 341d7ac..ca144f5 100644 --- a/StringIndex/StringIndex.cpp +++ b/StringIndex/StringIndex.cpp @@ -81,7 +81,7 @@ void StringIndexFile::load() this->empty_offset = max(this->empty_offset, (*this->index_table)[i].offset + (long)(*this->index_table)[i].length); } - trie->LoadTrie(dictionary_path); +// trie->LoadTrie(dictionary_path); } bool StringIndexFile::randomAccess(unsigned id, string *str, bool real) @@ -110,10 +110,10 @@ bool StringIndexFile::randomAccess(unsigned id, string *str, bool real) //cout<<"check: "<<*str<Uncompress(*str, str->length());//Uncompresss - } +// if (real) +// { +// *str = trie->Uncompress(*str, str->length());//Uncompresss +// } //if(id == 9) //{ //cout<<"check: "<<*str<buffer[length] = '\0'; *this->request[pos].str = string(this->buffer); - *this->request[pos].str = trie->Uncompress( - *this->request[pos].str, this->request[pos].str->length()); +// *this->request[pos].str = trie->Uncompress( +// *this->request[pos].str, this->request[pos].str->length()); pos++; } @@ -216,8 +216,8 @@ void StringIndexFile::trySequenceAccess(bool real) this->buffer[length] = '\0'; *this->request[pos].str += string(this->buffer); - *this->request[pos].str = trie->Uncompress( - *this->request[pos].str, this->request[pos].str->length()); +// *this->request[pos].str = trie->Uncompress( +// *this->request[pos].str, this->request[pos].str->length()); pos++; while (pos < (int)this->request.size() && this->request[pos - 1].offset == this->request[pos].offset) diff --git a/StringIndex/StringIndex.h b/StringIndex/StringIndex.h index 914486c..9277c2f 100644 --- a/StringIndex/StringIndex.h +++ b/StringIndex/StringIndex.h @@ -52,10 +52,10 @@ class StringIndexFile }; std::vector request; - std::string dictionary_path; +// std::string dictionary_path; public: - Trie *trie; + //Trie *trie; StringIndexFile(StringIndexFileType _type, std::string _dir, unsigned _num):type(_type), num(_num), empty_offset(0), index_file(NULL), value_file(NULL), buffer_size(0), buffer(NULL) { @@ -67,8 +67,8 @@ class StringIndexFile this->loc = _dir + "/predicate_"; this->index_table = new std::vector; - dictionary_path = _dir + "/../dictionary.dc"; - trie = new Trie; + //dictionary_path = _dir + "/../dictionary.dc"; + //trie = new Trie; } ~StringIndexFile() { @@ -79,8 +79,8 @@ class StringIndexFile if (this->buffer != NULL) delete[] this->buffer; delete this->index_table; - if (this->trie != NULL) - delete trie; +// if (this->trie != NULL) +// delete trie; } void clear() @@ -88,7 +88,7 @@ class StringIndexFile this->index_file = NULL; this->value_file = NULL; this->index_table = NULL; - this->trie = NULL; +// this->trie = NULL; } void setNum(unsigned _num); @@ -132,12 +132,12 @@ class StringIndex unsigned literal_buffer_size; public: - Trie *trie; +// Trie *trie; StringIndex(std::string _dir, unsigned _entity_num = 0, unsigned _literal_num = 0, unsigned _predicate_num = 0): entity(StringIndexFile::Entity, _dir, _entity_num), literal(StringIndexFile::Literal, _dir, _literal_num), predicate(StringIndexFile::Predicate, _dir, _predicate_num) { - trie = entity.trie; +// trie = entity.trie; } void clear()