opensci
/
gStore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'qinzongyue' into dev 

merge modifications of qinzongyue to indices
This commit is contained in:
bookug 2018-10-19 13:17:54 +08:00
parent f2c7e97655 0f0eb86ba9
commit a5e169afab
12 changed files with 124 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
Database/Database.cpp
View File

@ -51,7 +51,7 @@ Database::Database()
this->if_loaded = false;
this->trie = NULL;
// this->trie = NULL;
//this->resetIDinfo();
this->initIDinfo();
@ -103,7 +103,7 @@ Database::Database(string _name)
this->query_cache = new QueryCache();
this->trie = NULL;
// this->trie = NULL;
//this->resetIDinfo();
this->initIDinfo();
@ -685,8 +685,8 @@ Database::load()
return false;
}
if(!(this->kvstore)->load_trie())
return false;
// if(!(this->kvstore)->load_trie())
// return false;
//NOTICE: we should also run some heavy work in the main thread
this->stringindex->load();
@ -736,17 +736,17 @@ Database::load()
//this->warmUp();
//DEBUG:the warmUp() calls query(), which will also output results, this is not we want
// Load trie
// Load trie (trie has been removed)
if (trie != NULL)
delete trie;
trie = new Trie;
// if (trie != NULL)
// delete trie;
// trie = new Trie;
string dictionary_path = store_path + "/dictionary.dc";
if (!trie->LoadTrie(dictionary_path))
{
return false;
}
// string dictionary_path = store_path + "/dictionary.dc";
// if (!trie->LoadTrie(dictionary_path))
// {
// return false;
// }
this->if_loaded = true;
@ -1259,11 +1259,11 @@ Database::unload()
this->if_loaded = false;
this->clear_update_log();
if (this->trie != NULL)
{
delete this->trie;
trie = NULL;
}
// if (this->trie != NULL)
// {
// delete this->trie;
// trie = NULL;
// }
return true;
}
@ -1395,7 +1395,7 @@ Database::get_query_parse_lock()
int
Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
{
string dictionary_store_path = this->store_path + "/dictionary.dc";
// string dictionary_store_path = this->store_path + "/dictionary.dc";
GeneralEvaluation general_evaluation(this->vstree, this->kvstore, this->stringindex, this->query_cache, this->pre2num, this->limitID_predicate, this->limitID_literal,this->limitID_entity);
@ -1444,16 +1444,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
long tv_bfget = Util::get_cur_time();
//NOTICE: this lock lock ensures that StringIndex is visited sequentially
this->getFinalResult_lock.lock();
if (trie == NULL)
{
trie = new Trie;
string dictionary_path = this->store_path + "/dictionary.dc";
if (!trie->LoadTrie(dictionary_path))
{
exit(0);
}
trie->LoadDictionary();
}
// if (trie == NULL)
// {
// trie = new Trie;
// string dictionary_path = this->store_path + "/dictionary.dc";
// if (!trie->LoadTrie(dictionary_path))
// {
// exit(0);
// }
// trie->LoadDictionary();
// }
general_evaluation.getFinalResult(_result_set);
this->getFinalResult_lock.unlock();
long tv_afget = Util::get_cur_time();
@ -1485,16 +1485,16 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
TripleWithObjType *update_triple = NULL;
TYPE_TRIPLE_NUM update_triple_num = 0;
if (trie == NULL)
{
trie = new Trie;
string dictionary_path = this->store_path + "/dictionary.dc";
if (!trie->LoadTrie(dictionary_path))
{
exit(0);
}
trie->LoadDictionary();
}
// if (trie == NULL)
// {
// trie = new Trie;
// string dictionary_path = this->store_path + "/dictionary.dc";
// if (!trie->LoadTrie(dictionary_path))
// {
// exit(0);
// }
// trie->LoadDictionary();
// }
if (general_evaluation.getQueryTree().getUpdateType() == QueryTree::Insert_Data || general_evaluation.getQueryTree().getUpdateType() == QueryTree::Delete_Data)
@ -1519,7 +1519,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
update_pattern.sub_group_pattern[i].pattern.object.value, object_type);
// Compress
update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE);
// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE);
}
else
{
@ -1547,7 +1547,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getDeletePatterns(), update_triple, update_triple_num);
for(int i = 0; i < update_triple_num; i++)
{
update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE);
// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE);
}
success_num = remove(update_triple, update_triple_num);
}
@ -1556,7 +1556,7 @@ Database::query(const string _query, ResultSet& _result_set, FILE* _fp)
general_evaluation.prepareUpdateTriple(general_evaluation.getQueryTree().getInsertPatterns(), update_triple, update_triple_num);
for(int i = 0; i < update_triple_num; i++)
{
update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE);
// update_triple[i] = trie->Compress(update_triple[i], Trie::QUERYMODE);
}
success_num = insert(update_triple, update_triple_num);
}
@ -2413,7 +2413,7 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file)
(this->kvstore)->open_id2predicate(KVstore::CREATE_MODE);
(this->kvstore)->open_literal2id(KVstore::CREATE_MODE);
(this->kvstore)->open_id2literal(KVstore::CREATE_MODE);
(this->kvstore)->load_trie();
// (this->kvstore)->load_trie();
}
//Util::logging("finish initial sub2id_pre2id_obj2id");
@ -2457,19 +2457,19 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file)
RDFParser _parser(_fin);
// Initialize trie
string dictionary_store_path = this->store_path + "/dictionary.dc";
this->trie = new Trie (_rdf_file, dictionary_store_path);
if (!trie->isInitialized())
{
cout << "Fail to initialize trie" << endl;
exit(0);
}
// string dictionary_store_path = this->store_path + "/dictionary.dc";
// this->trie = new Trie (_rdf_file, dictionary_store_path);
// if (!trie->isInitialized())
// {
// cout << "Fail to initialize trie" << endl;
// exit(0);
// }
if(!trie->WriteDown())
{
cout << "Fail to write down dictionary" << endl;
exit(0);
}
// if(!trie->WriteDown())
// {
// cout << "Fail to write down dictionary" << endl;
// exit(0);
// }
//Util::logging("==> while(true)");
while (true)
@ -2515,8 +2515,8 @@ Database::sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file)
//We need a thread pool!
// Compress triple begin
TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE);
triple_array[i] = compressed_triple;
// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::BUILDMODE);
// triple_array[i] = compressed_triple;
//Compress triple end
// For subject
@ -3036,11 +3036,11 @@ Database::insert(std::string _rdf_file, bool _is_restore)
}
//Compress triple here
for(int i = 0; i < parse_triple_num; i++)
{
TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE);
triple_array[i] = compressed_triple;
}
// for(int i = 0; i < parse_triple_num; i++)
// {
// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE);
// triple_array[i] = compressed_triple;
// }
//Process the Triple one by one
success_num += this->insert(triple_array, parse_triple_num, _is_restore);
@ -3136,11 +3136,11 @@ Database::remove(std::string _rdf_file, bool _is_restore)
}
//Compress triple
for(int i = 0; i < parse_triple_num; i++)
{
TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE);
triple_array[i] = compressed_triple;
}
// for(int i = 0; i < parse_triple_num; i++)
// {
// TripleWithObjType compressed_triple = trie->Compress(triple_array[i], Trie::QUERYMODE);
// triple_array[i] = compressed_triple;
// }
success_num += this->remove(triple_array, parse_triple_num, _is_restore);

2
Database/Database.h
View File

@ -155,7 +155,7 @@ private:
QueryCache *query_cache;
Trie *trie;
//Trie *trie;
void setStringBuffer();
void warmUp();

4
KVstore/ISArray/ISArray.cpp
View File

@ -16,7 +16,6 @@ ISArray::ISArray()
dir_path = "";
ISfile_name = "";
BM = NULL;
CurKeyNum = 0;
CurEntryNum = 0;
CurCacheSize = 0;
CurEntryNumChange = false;
@ -55,7 +54,6 @@ ISArray::ISArray(string _dir_path, string _filename, string mode, unsigned long
// temp is the smallest number >= _key_num and mod SET_KEY_INC = 0
unsigned temp = ((_key_num + (1 << 10) - 1) >> 10) << 10;
CurKeyNum = 0;
CurEntryNum = max(temp, SETKEYNUM);
CurEntryNumChange = true;
@ -341,7 +339,6 @@ ISArray::insert(unsigned _key, char *_str, unsigned _len)
}
bool Alloc = false;
CurKeyNum++;
//if (CurKeyNum >= CurEntryNum) // need to realloc
if (_key >= CurEntryNum)
{
@ -391,7 +388,6 @@ ISArray::remove(unsigned _key)
return false;
}
CurKeyNum--;
unsigned store = array[_key].getStore();
BM->FreeBlocks(store);

1
KVstore/ISArray/ISArray.h
View File

@ -35,7 +35,6 @@ private:
string dir_path;
ISBlockManager *BM;
unsigned CurEntryNum; // how many entries are available
unsigned CurKeyNum; // how many keys are stored
bool CurEntryNumChange;
//Cache

8
KVstore/ISArray/ISBlockManager.cpp
View File

@ -335,8 +335,10 @@ ISBlockManager::FreeBlocks(const unsigned index)
{
cur_index = it->first;
curlen += it->second;
auto tmp_key = it->second, tmp_val = it->first;
index_len_map.erase(it);
len_index_map.erase(make_pair(it->second,it->first));
//len_index_map.erase(make_pair(it->second,it->first));
len_index_map.erase(make_pair(tmp_key, tmp_val));
}
}
@ -346,8 +348,10 @@ ISBlockManager::FreeBlocks(const unsigned index)
if (curlen + cur_index == it->first) // block after is free
{
curlen += it->second;
auto tmp_key = it->second, tmp_val = it->first;
index_len_map.erase(it);
len_index_map.erase(make_pair(it->second,it->first));
//len_index_map.erase(make_pair(it->second,it->first));
len_index_map.erase(make_pair(tmp_key, tmp_val));
}
}

4
KVstore/IVArray/IVArray.cpp
View File

@ -16,7 +16,6 @@ IVArray::IVArray()
dir_path = "";
IVfile_name = "";
BM = NULL;
CurKeyNum = 0;
CurEntryNum = 0;
CurCacheSize = 0;
CurEntryNumChange = false;
@ -63,7 +62,6 @@ IVArray::IVArray(string _dir_path, string _filename, string mode, unsigned long
// temp is the smallest number >= _key_num and mod SET_KEY_INC = 0
unsigned temp = ((_key_num + (1 << 10) - 1) >> 10) << 10;
CurKeyNum = 0;
CurEntryNum = max(temp, SETKEYNUM);
CurEntryNumChange = true;
@ -346,7 +344,6 @@ IVArray::insert(unsigned _key, char *_str, unsigned _len)
return false;
}
CurKeyNum++;
//if (CurKeyNum >= CurEntryNum) // need to realloc
if (_key >= CurEntryNum)
{
@ -402,7 +399,6 @@ IVArray::remove(unsigned _key)
return false;
}
CurKeyNum--;
unsigned store = array[_key].getStore();
BM->FreeBlocks(store);

1
KVstore/IVArray/IVArray.h
View File

@ -36,7 +36,6 @@ private:
string dir_path;
IVBlockManager *BM;
unsigned CurEntryNum; // how many entries are available
unsigned CurKeyNum; // how many keys are stored
bool CurEntryNumChange;
//Cache

4
KVstore/IVArray/IVBlockManager.cpp
View File

@ -335,8 +335,10 @@ IVBlockManager::FreeBlocks(const unsigned index)
{
cur_index = it->first;
curlen += it->second;
auto tmp_key = it->second, tmp_val = it->first;
index_len_map.erase(it);
len_index_map.erase(make_pair(it->second,it->first));
//len_index_map.erase(make_pair(it->second,it->first));
len_index_map.erase(make_pair(tmp_key, tmp_val));
}
}

50
KVstore/KVstore.cpp
View File

@ -16,8 +16,8 @@ KVstore::KVstore(string _store_path)
{
this->store_path = _store_path;
this->dictionary_store_path = _store_path + "/../dictionary.dc";
this->trie = NULL;
// this->dictionary_store_path = _store_path + "/../dictionary.dc";
//this->trie = NULL;
this->entity2id = NULL;
this->id2entity = NULL;
@ -97,9 +97,9 @@ KVstore::release()
delete this->objID2values_literal;
this->objID2values_literal = NULL;
if (trie != NULL)
delete this->trie;
this->trie = NULL;
//if (trie != NULL)
// delete this->trie;
//this->trie = NULL;
}
void
@ -120,18 +120,18 @@ KVstore::open()
this->open_objID2values(KVstore::READ_WRITE_MODE);
this->open_preID2values(KVstore::READ_WRITE_MODE);
this->trie = new Trie;
trie->LoadTrie(dictionary_store_path);
//this->trie = new Trie;
//trie->LoadTrie(dictionary_store_path);
}
bool
KVstore::load_trie()
{
if (trie != NULL) return true;
//bool
//KVstore::load_trie()
//{
// if (trie != NULL) return true;
trie = new Trie;
return trie->LoadTrie(dictionary_store_path);
}
// trie = new Trie;
// return trie->LoadTrie(dictionary_store_path);
//}
string
KVstore::getStringByID(TYPE_ENTITY_LITERAL_ID _id)
@ -1099,7 +1099,7 @@ KVstore::subIDByEntity(string _entity)
//_entity will not be released befor ethis function ends
//so _entity.c_str() is a valid const char*
//this->load_trie();
_entity = trie->Compress(_entity);
//_entity = trie->Compress(_entity);
//return this->entity2id->remove(_entity.c_str(), _entity.length());
return this->removeKey(this->entity2id, _entity.c_str(), _entity.length());
}
@ -1108,7 +1108,7 @@ TYPE_ENTITY_LITERAL_ID
KVstore::getIDByEntity(string _entity) const
{
//this->load_trie();
_entity = trie->Compress(_entity);
//_entity = trie->Compress(_entity);
return this->getIDByStr(this->entity2id, _entity.c_str(), _entity.length());
}
@ -1121,7 +1121,7 @@ KVstore::setIDByEntity(string _entity, TYPE_ENTITY_LITERAL_ID _id)
char* str = new char[len];
//this->load_trie();
_entity = trie->Compress(_entity);
//_entity = trie->Compress(_entity);
memcpy(str, _entity.c_str(), len);
return this->addValueByKey(this->entity2id, str, len, _id);
}
@ -1196,7 +1196,7 @@ KVstore::getEntityByID(TYPE_ENTITY_LITERAL_ID _id) const
//cout << _tmp << endl;
string _ret = string(_tmp, _len);
_ret = trie->Uncompress(_ret, _ret.length());
// _ret = trie->Uncompress(_ret, _ret.length());
delete []_tmp;
// cout << "string is: " << _ret << endl;
// string test1 = _ret;
@ -1260,7 +1260,7 @@ bool
KVstore::subIDByPredicate(string _predicate)
{
//this->load_trie();
_predicate = trie->Compress(_predicate);
// _predicate = trie->Compress(_predicate);
//return this->predicate2id->remove(_predicate.c_str(), _predicate.length());
return this->removeKey(this->predicate2id, _predicate.c_str(), _predicate.length());
}
@ -1269,7 +1269,7 @@ TYPE_PREDICATE_ID
KVstore::getIDByPredicate(string _predicate) const
{
//this->load_trie();
_predicate = trie->Compress(_predicate);
// _predicate = trie->Compress(_predicate);
return this->getIDByStr(this->predicate2id, _predicate.c_str(), _predicate.length());
}
@ -1282,7 +1282,7 @@ KVstore::setIDByPredicate(string _predicate, TYPE_PREDICATE_ID _id)
char* str = new char[len];
//this->load_trie();
_predicate = trie->Compress(_predicate);
// _predicate = trie->Compress(_predicate);
memcpy(str, _predicate.c_str(), len);
return this->addValueByKey(this->predicate2id, str, len, _id);
}
@ -1402,7 +1402,7 @@ bool
KVstore::subIDByLiteral(string _literal)
{
//this->load_trie();
_literal = trie->Compress(_literal);
// _literal = trie->Compress(_literal);
//return this->literal2id->remove(_literal.c_str(), _literal.length());
return this->removeKey(this->literal2id, _literal.c_str(), _literal.length());
}
@ -1411,7 +1411,7 @@ TYPE_ENTITY_LITERAL_ID
KVstore::getIDByLiteral(string _literal) const
{
//this->load_trie();
_literal = trie->Compress(_literal);
// _literal = trie->Compress(_literal);
return this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length());
//TYPE_ENTITY_LITERAL_ID id = this->getIDByStr(this->literal2id, _literal.c_str(), _literal.length());
//if(id != INVALID)
@ -1428,7 +1428,7 @@ KVstore::setIDByLiteral(string _literal, TYPE_ENTITY_LITERAL_ID _id)
//this->load_trie();
int len = _literal.length();
char* str = new char[len];
_literal = trie->Compress(_literal);
// _literal = trie->Compress(_literal);
memcpy(str, _literal.c_str(), len);
return this->addValueByKey(this->literal2id, str, len, _id);
@ -1493,7 +1493,7 @@ KVstore::getLiteralByID(TYPE_ENTITY_LITERAL_ID _id) const
}
string _ret = string(_tmp, _len);
delete []_tmp;
_ret = trie->Uncompress(_ret, _ret.length());
// _ret = trie->Uncompress(_ret, _ret.length());
return _ret;
}

8
KVstore/KVstore.h
View File

@ -12,7 +12,7 @@
#include "../Util/Util.h"
#include "../Util/VList.h"
#include "Tree.h"
#include "../Trie/Trie.h"
//#include "../Trie/Trie.h"
#include "IVArray/IVArray.h"
#include "ISArray/ISArray.h"
@ -169,14 +169,14 @@ public:
//for so2p
bool getpreIDlistBysubIDobjID(TYPE_ENTITY_LITERAL_ID _subID, TYPE_ENTITY_LITERAL_ID _objID, unsigned*& _preidlist, unsigned& _list_len, bool _no_duplicate = false) const;
bool load_trie();
//bool load_trie();
private:
std::string store_path;
std::string dictionary_store_path;
// std::string dictionary_store_path;
Trie *trie;
//Trie *trie;
SITree* entity2id;
//ISTree* id2entity;

18
StringIndex/StringIndex.cpp
View File

@ -81,7 +81,7 @@ void StringIndexFile::load()
this->empty_offset = max(this->empty_offset, (*this->index_table)[i].offset + (long)(*this->index_table)[i].length);
}
trie->LoadTrie(dictionary_path);
// trie->LoadTrie(dictionary_path);
}
bool StringIndexFile::randomAccess(unsigned id, string *str, bool real)
@ -110,10 +110,10 @@ bool StringIndexFile::randomAccess(unsigned id, string *str, bool real)
//cout<<"check: "<<*str<<endl;
//}
if (real)
{
*str = trie->Uncompress(*str, str->length());//Uncompresss
}
// if (real)
// {
// *str = trie->Uncompress(*str, str->length());//Uncompresss
// }
//if(id == 9)
//{
//cout<<"check: "<<*str<<endl;
@ -194,8 +194,8 @@ void StringIndexFile::trySequenceAccess(bool real)
this->buffer[length] = '\0';
*this->request[pos].str = string(this->buffer);
*this->request[pos].str = trie->Uncompress(
*this->request[pos].str, this->request[pos].str->length());
// *this->request[pos].str = trie->Uncompress(
// *this->request[pos].str, this->request[pos].str->length());
pos++;
}
@ -216,8 +216,8 @@ void StringIndexFile::trySequenceAccess(bool real)
this->buffer[length] = '\0';
*this->request[pos].str += string(this->buffer);
*this->request[pos].str = trie->Uncompress(
*this->request[pos].str, this->request[pos].str->length());
// *this->request[pos].str = trie->Uncompress(
// *this->request[pos].str, this->request[pos].str->length());
pos++;
while (pos < (int)this->request.size() && this->request[pos - 1].offset == this->request[pos].offset)

18
StringIndex/StringIndex.h
View File

@ -52,10 +52,10 @@ class StringIndexFile
};
std::vector<AccessRequest> request;
std::string dictionary_path;
// std::string dictionary_path;
public:
Trie *trie;
//Trie *trie;
StringIndexFile(StringIndexFileType _type, std::string _dir, unsigned _num):type(_type), num(_num), empty_offset(0), index_file(NULL), value_file(NULL), buffer_size(0), buffer(NULL)
{
@ -67,8 +67,8 @@ class StringIndexFile
this->loc = _dir + "/predicate_";
this->index_table = new std::vector<IndexInfo>;
dictionary_path = _dir + "/../dictionary.dc";
trie = new Trie;
//dictionary_path = _dir + "/../dictionary.dc";
//trie = new Trie;
}
~StringIndexFile()
{
@ -79,8 +79,8 @@ class StringIndexFile
if (this->buffer != NULL)
delete[] this->buffer;
delete this->index_table;
if (this->trie != NULL)
delete trie;
// if (this->trie != NULL)
// delete trie;
}
void clear()
@ -88,7 +88,7 @@ class StringIndexFile
this->index_file = NULL;
this->value_file = NULL;
this->index_table = NULL;
this->trie = NULL;
// this->trie = NULL;
}
void setNum(unsigned _num);
@ -132,12 +132,12 @@ class StringIndex
unsigned literal_buffer_size;
public:
Trie *trie;
// Trie *trie;
StringIndex(std::string _dir, unsigned _entity_num = 0, unsigned _literal_num = 0, unsigned _predicate_num = 0):
entity(StringIndexFile::Entity, _dir, _entity_num), literal(StringIndexFile::Literal, _dir, _literal_num), predicate(StringIndexFile::Predicate, _dir, _predicate_num)
{
trie = entity.trie;
// trie = entity.trie;
}
void clear()