opensci
/
gStore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: merge http branch

This commit is contained in:
bookug 2017-05-18 22:56:41 +08:00
parent 25e1c3ed10 d1d85deaa2
commit 77499f990e
18 changed files with 1629 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Database/Join.cpp
View File

@ -1268,6 +1268,8 @@ Join::multi_join()
for (unsigned i = 0; i < start_size; ++i)
{
unsigned ele = start_table.getID(i);
//NOTICE: we can denote the total size here in vector, but no need because the variables' num is small
//(won't double to require more space)
RecordType record(1, ele);
this->current_table.push_back(record);
//this->table_row_new.push_back(false);
@ -1419,8 +1421,16 @@ Join::filter_before_join()
IDList &can_list = this->basic_query->getCandidateList(i);
//fprintf(stderr, "\t\tsize of canlist before filter: %d\n", can_list.size());
cout << "\t\tsize of canlist before filter: " << can_list.size() << endl;
//NOTICE:must sort before using binary search.
//However, the sort-merge maybe not always better because the sort() will take too much time if
//the can_list size is large, i.e. > 1000000
can_list.sort();
//vstree ? place on ID?
//TODO: use BoolArray isntead of bitset
//n is candidate num, m is sp2o num, then when n<m/(lg2(m)_lg2(n)), sort m and binary search in m
//otherwise, use BoolArray for n, only construct a time
//NOTICE: for parallelism, use a BoolArray for each BGP(either on join or in Strategy)
long begin = Util::get_cur_time();
bool ret = this->constant_edge_filter(i);

322
Main/HttpConnector.cpp Executable file
View File

@ -0,0 +1,322 @@
#include "../Server/server_http.hpp"
#include "../Server/client_http.hpp"
//Added for the json-example
#define BOOST_SPIRIT_THREADSAFE
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
//Added for the default_resource example
#include <fstream>
#include <boost/filesystem.hpp>
//#include <boost/regex.hpp>
#include <vector>
#include <algorithm>
#include <memory>
//db
#include "../Database/Database.h"
#include "../Util/Util.h"
using namespace std;
//Added for the json-example:
using namespace boost::property_tree;
typedef SimpleWeb::Server<SimpleWeb::HTTP> HttpServer;
typedef SimpleWeb::Client<SimpleWeb::HTTP> HttpClient;
//Added for the default_resource example
void default_resource_send(const HttpServer &server, const shared_ptr<HttpServer::Response> &response,
const shared_ptr<ifstream> &ifs);
Database *current_database = NULL;
int main() {
Util util;
//HTTP-server at port 8080 using 1 thread
//Unless you do more heavy non-threaded processing in the resources,
//1 thread is usually faster than several threads
HttpServer server;
server.config.port=8080;
//server.config.port=9000;
//cout<<"after server built"<<endl;
//GET-example for the path /build/[db_name]/[db_path], responds with the matched string in path
//For instance a request GET /build/db/123 will receive: db 123
//server.resource["^/build/([a-zA-Z]+[0-9]*)/([a-zA-Z]+/*[a-zA-Z]+[0-9]*.n[a-zA-Z]*[0-9]*)$"]["GET"]=[&server](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) {
// server.resource["^/build/([a-zA-Z0-9]+)/([a-zA-Z0-9]+)$"]["GET"]=[&server](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) {
server.resource["^/build/([a-zA-Z0-9]*)/(.*)$"]["GET"]=[&server](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) {
string db_name=request->path_match[1];
string db_path=request->path_match[2];
if(db_name=="" || db_path=="")
{
string error = "Exactly 2 arguments required!";
// error = db_name + " " + db_path;
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
}
string database = db_name;
if(database.length() > 3 && database.substr(database.length()-3, 3) == ".db")
{
string error = "Your db name to be built should not end with \".db\".";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
}
//database += ".db";
string dataset = db_path;
if(current_database != NULL)
{
string error = "Please unload your database first.";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
}
cout << "Import dataset to build database..." << endl;
cout << "DB_store: " << database << "\tRDF_data: " << dataset << endl;
int len = database.length();
current_database = new Database(database);
bool flag = current_database->build(dataset);
delete current_database;
current_database = NULL;
if(!flag)
{
string error = "Import RDF file to database failed.";
string cmd = "rm -r " + database;
system(cmd.c_str());
return 0;
}
// string success = db_name + " " + db_path;
string success = "Import RDF file to database done.";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success;
return 0;
};
//GET-example for the path /load/[db_name], responds with the matched string in path
//For instance a request GET /load/db123 will receive: db123
server.resource["^/load/(.*)$"]["GET"]=[&server](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) {
string db_name=request->path_match[1];
if(db_name=="")
{
string error = "Exactly 1 argument is required!";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
}
string database = db_name;
if(database.length() > 3 && database.substr(database.length()-3, 3) == ".db")
{
string error = "Your db name to be built should not end with \".db\".";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
}
//database += ".db";
if(current_database != NULL)
{
string error = "Please unload your current database first.";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
}
cout << database << endl;
current_database = new Database(database);
bool flag = current_database->load();
if (!flag)
{
string error = "Failed to load the database.";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
delete current_database;
current_database = NULL;
return 0;
}
//string success = db_name;
string success = "Database loaded successfully.";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success;
return 0;
};
//GET-example for the path /query/[query_file_path], responds with the matched string in path
//For instance a request GET /query/db123 will receive: db123
server.resource["^/query/(.*)$"]["GET"] = [&server](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) {
string db_query=request->path_match[1];
string str = db_query;
if(current_database == NULL)
{
string error = "No database in use!";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
};
string sparql;
if(db_query[0]=='\"')
{
sparql = db_query.substr(1, db_query.length()-2);
}
else
{
string ret = Util::getExactPath(db_query.c_str());
const char *path = ret.c_str();
if(path == NULL)
{
string error = "Invalid path of query.";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
}
sparql = Util::getQueryFromFile(path);
}
if (sparql.empty()) {
cerr << "Empty SPARQL." << endl;
return 0;
}
FILE* output = stdout;
ResultSet rs;
bool ret = current_database->query(sparql, rs, output);
if(ret)
{
string success = rs.to_str();
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success;
return 0;
}
else
{
string error = "query() returns false.";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
}
};
//GET-example for the path /unload/[db_name], responds with the matched string in path
//For instance a request GET /unload/db123 will receive: db123
server.resource["^/unload$"]["GET"]=[&server](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) {
if(current_database == NULL)
{
string error = "No database used now.";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
return 0;
}
delete current_database;
current_database = NULL;
string success = "Database unloaded.";
*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success;
return 0;
};
// server.resource["^/json$"]["POST"]=[](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) {
// try {
// ptree pt;
// read_json(request->content, pt);
// string name=pt.get<string>("firstName")+" "+pt.get<string>("lastName");
// *response << "HTTP/1.1 200 OK\r\n"
// << "Content-Type: application/json\r\n"
// << "Content-Length: " << name.length() << "\r\n\r\n"
// << name;
// }
// catch(exception& e) {
// *response << "HTTP/1.1 400 Bad Request\r\nContent-Length: " << strlen(e.what()) << "\r\n\r\n" << e.what();
// }
// };
//Default GET-example. If no other matches, this anonymous function will be called.
//Will respond with content in the web/-directory, and its subdirectories.
//Default file: index.html
//Can for instance be used to retrieve an HTML 5 client that uses REST-resources on this server
server.default_resource["GET"]=[&server](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) {
try {
auto web_root_path=boost::filesystem::canonical("./Server/web");
auto path=boost::filesystem::canonical(web_root_path/request->path);
//Check if path is within web_root_path
if(distance(web_root_path.begin(), web_root_path.end())>distance(path.begin(), path.end()) ||
!equal(web_root_path.begin(), web_root_path.end(), path.begin()))
throw invalid_argument("path must be within root path");
if(boost::filesystem::is_directory(path))
path/="index.html";
if(!(boost::filesystem::exists(path) && boost::filesystem::is_regular_file(path)))
throw invalid_argument("file does not exist");
std::string cache_control, etag;
// Uncomment the following line to enable Cache-Control
// cache_control="Cache-Control: max-age=86400\r\n";
auto ifs=make_shared<ifstream>();
ifs->open(path.string(), ifstream::in | ios::binary | ios::ate);
if(*ifs) {
auto length=ifs->tellg();
ifs->seekg(0, ios::beg);
*response << "HTTP/1.1 200 OK\r\n" << cache_control << etag << "Content-Length: " << length << "\r\n\r\n";
default_resource_send(server, response, ifs);
}
else
throw invalid_argument("could not read file");
}
catch(const exception &e) {
string content="Could not open path "+request->path+": "+e.what();
*response << "HTTP/1.1 400 Bad Request\r\nContent-Length: " << content.length() << "\r\n\r\n" << content;
}
};
thread server_thread([&server](){
//Start server
server.start();
});
//Wait for server to start so that the client can connect
this_thread::sleep_for(chrono::seconds(1));
// //Client examples
// HttpClient client("localhost:8080");
// auto r1=client.request("GET", "/match/123");
// cout << r1->content.rdbuf() << endl;
// string json_string="{\"firstName\": \"John\",\"lastName\": \"Smith\",\"age\": 25}";
// auto r2=client.request("POST", "/string", json_string);
// cout << r2->content.rdbuf() << endl;
// auto r3=client.request("POST", "/json", json_string);
// cout << r3->content.rdbuf() << endl;
server_thread.join();
return 0;
}
void default_resource_send(const HttpServer &server, const shared_ptr<HttpServer::Response> &response,
const shared_ptr<ifstream> &ifs) {
//read and send 128 KB at a time
static vector<char> buffer(131072); // Safe when server is running on one thread
streamsize read_length;
if((read_length=ifs->read(&buffer[0], buffer.size()).gcount())>0) {
response->write(&buffer[0], read_length);
if(read_length==static_cast<streamsize>(buffer.size())) {
server.send(response, [&server, response, ifs](const boost::system::error_code &ec) {
if(!ec)
default_resource_send(server, response, ifs);
else
cerr << "Connection interrupted" << endl;
});
}
}
}

7
NOTES.md
View File

@ -94,6 +94,11 @@ http://blog.csdn.net/infoworld/article/details/8670951
type分支中query过程可能还有问题，需要修改Query/里面的类型，另外stringindex中也要修改，分界线已经是20亿且非法不再是-1
remove signature.binary, 合并两个分支type value
vstree在build和query时可以用不同大小的缓存，来加速build过程
vstree建立太耗时，严重拖了build过程的后腿
triple num改为unsigned long long，争取单机最大支持到100亿数据集，只要entity等数目不超过20亿。
同时将ID的编码改为unsigned，无效标志-1改为最大值的宏, triple数目的类型也要改为unsigned
注意pre的ID还可以为-2，或者对于pre仍然用int，或者改函数的返回值为long long (还有一些没有用-1而是>=0)
---
将B+tree中叶节点的大的value分离出来，新建一套缓存，使用block机制，标记length为0表示未读取
类型bstr的length问题也需要解决(新建Istr类型)
@ -606,6 +611,8 @@ ACID? neo4j GraphDB
## 单个文件的gStore？嵌入式，轻便，类似sqlite，方便移植，做成库的方式给python等调用
## 联邦数据库，避免数据重导入，上层查询分块
mysql适合存属性，而gstore适合处理关系，trinity适合做算法
gstore是否能处理各种图算法需求呢，比如对两点求最短路？比如正则路径查询？
## 没必要关闭IO缓冲同步，因为用的基本都是C语言的输入输出操作

4
README.md
View File

@ -4,7 +4,7 @@ Gstore System(also called gStore) is a graph database engine for managing large
**The formal help document is in [Handbook](docs/latex/gStore_help.pdf).**
**We have built an IRC channel named #gStore on freenode, and you can also come to our website: [gStore](gstore-pku.com).**
**We have built an IRC channel named #gStore on freenode, and you can also come to our website: [gStore](http://gstore-pku.com).**
<!--**You can write your information in [survey](http://59.108.48.38/survey) if you like.**-->
@ -48,7 +48,7 @@ If you want to understand the details of the gStore system, or you want to try s
We have written a series of short essays addressing recurring challenges in using gStore to realize applications, which are placed in [Recipe Book](docs/TIPS.md).
You are welcome to report any advice or errors in the github Issues part of this repository, if not requiring in-time reply. However, if you want to urgent on us to deal with your reports, please email to <chenjiaqi93@163.com> to submit your suggestions and report bugs to us by emailing to <zengli-bookug@pku.edu.cn>. A full list of our whole team is in [Mailing List](docs/MAIL.md).
You are welcome to report any advice or errors in the github Issues part of this repository, if not requiring in-time reply. However, if you want to urgent on us to deal with your reports, please email to <zengli@bookug.cc> to submit your suggestions and report bugs to us by emailing to <gStoreDB@gmail.com>. A full list of our whole team is in [Mailing List](docs/MAIL.md).
There are some restrictions when you use the current gStore project, you can see them on [Limit Description](docs/LIMIT.md).

21
Server/LICENSE Executable file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014-2016 Ole Christian Eidheim
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

8
Server/README.md Executable file
View File

@ -0,0 +1,8 @@
A Http interface for gStore
=================
Based on Simple-Web-Server (https://travis-ci.org/eidheim/Simple-Web-Server)
To see the whole project about gStore, click at https://github.com/Caesar11/gStore

440
Server/client_http.hpp Executable file
View File

@ -0,0 +1,440 @@
#ifndef CLIENT_HTTP_HPP
#define CLIENT_HTTP_HPP
#include <boost/asio.hpp>
#include <boost/utility/string_ref.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/functional/hash.hpp>
#include <unordered_map>
#include <map>
#include <random>
#include <mutex>
#include <type_traits>
#ifndef CASE_INSENSITIVE_EQUALS_AND_HASH
#define CASE_INSENSITIVE_EQUALS_AND_HASH
//Based on http://www.boost.org/doc/libs/1_60_0/doc/html/unordered/hash_equality.html
class case_insensitive_equals {
public:
bool operator()(const std::string &key1, const std::string &key2) const {
return boost::algorithm::iequals(key1, key2);
}
};
class case_insensitive_hash {
public:
size_t operator()(const std::string &key) const {
std::size_t seed=0;
for(auto &c: key)
boost::hash_combine(seed, std::tolower(c));
return seed;
}
};
#endif
namespace SimpleWeb {
template <class socket_type>
class Client;
template <class socket_type>
class ClientBase {
public:
virtual ~ClientBase() {}
class Response {
friend class ClientBase<socket_type>;
friend class Client<socket_type>;
public:
std::string http_version, status_code;
std::istream content;
std::unordered_multimap<std::string, std::string, case_insensitive_hash, case_insensitive_equals> header;
private:
boost::asio::streambuf content_buffer;
Response(): content(&content_buffer) {}
};
class Config {
friend class ClientBase<socket_type>;
private:
Config() {}
public:
/// Set timeout on requests in seconds. Default value: 0 (no timeout).
size_t timeout=0;
/// Set connect timeout in seconds. Default value: 0 (Config::timeout is then used instead).
size_t timeout_connect=0;
/// Set proxy server (server:port)
std::string proxy_server;
};
/// Set before calling request
Config config;
std::shared_ptr<Response> request(const std::string& request_type, const std::string& path="/", boost::string_ref content="",
const std::map<std::string, std::string>& header=std::map<std::string, std::string>()) {
auto corrected_path=path;
if(corrected_path=="")
corrected_path="/";
if(!config.proxy_server.empty() && std::is_same<socket_type, boost::asio::ip::tcp::socket>::value)
corrected_path="http://"+host+':'+std::to_string(port)+corrected_path;
boost::asio::streambuf write_buffer;
std::ostream write_stream(&write_buffer);
write_stream << request_type << " " << corrected_path << " HTTP/1.1\r\n";
write_stream << "Host: " << host << "\r\n";
for(auto& h: header) {
write_stream << h.first << ": " << h.second << "\r\n";
}
if(content.size()>0)
write_stream << "Content-Length: " << content.size() << "\r\n";
write_stream << "\r\n";
connect();
auto timer=get_timeout_timer();
boost::asio::async_write(*socket, write_buffer,
[this, &content, timer](const boost::system::error_code &ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(!ec) {
if(!content.empty()) {
auto timer=get_timeout_timer();
boost::asio::async_write(*socket, boost::asio::buffer(content.data(), content.size()),
[this, timer](const boost::system::error_code &ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(ec) {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
io_service.reset();
io_service.run();
return request_read();
}
std::shared_ptr<Response> request(const std::string& request_type, const std::string& path, std::iostream& content,
const std::map<std::string, std::string>& header=std::map<std::string, std::string>()) {
auto corrected_path=path;
if(corrected_path=="")
corrected_path="/";
if(!config.proxy_server.empty() && std::is_same<socket_type, boost::asio::ip::tcp::socket>::value)
corrected_path="http://"+host+':'+std::to_string(port)+corrected_path;
content.seekp(0, std::ios::end);
auto content_length=content.tellp();
content.seekp(0, std::ios::beg);
boost::asio::streambuf write_buffer;
std::ostream write_stream(&write_buffer);
write_stream << request_type << " " << corrected_path << " HTTP/1.1\r\n";
write_stream << "Host: " << host << "\r\n";
for(auto& h: header) {
write_stream << h.first << ": " << h.second << "\r\n";
}
if(content_length>0)
write_stream << "Content-Length: " << content_length << "\r\n";
write_stream << "\r\n";
if(content_length>0)
write_stream << content.rdbuf();
connect();
auto timer=get_timeout_timer();
boost::asio::async_write(*socket, write_buffer,
[this, timer](const boost::system::error_code &ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(ec) {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
io_service.reset();
io_service.run();
return request_read();
}
void close() {
std::lock_guard<std::mutex> lock(socket_mutex);
if(socket) {
boost::system::error_code ec;
socket->lowest_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);
socket->lowest_layer().close();
}
}
protected:
boost::asio::io_service io_service;
boost::asio::ip::tcp::resolver resolver;
std::unique_ptr<socket_type> socket;
std::mutex socket_mutex;
std::string host;
unsigned short port;
ClientBase(const std::string& host_port, unsigned short default_port) : resolver(io_service) {
auto parsed_host_port=parse_host_port(host_port, default_port);
host=parsed_host_port.first;
port=parsed_host_port.second;
}
std::pair<std::string, unsigned short> parse_host_port(const std::string &host_port, unsigned short default_port) {
std::pair<std::string, unsigned short> parsed_host_port;
size_t host_end=host_port.find(':');
if(host_end==std::string::npos) {
parsed_host_port.first=host_port;
parsed_host_port.second=default_port;
}
else {
parsed_host_port.first=host_port.substr(0, host_end);
parsed_host_port.second=static_cast<unsigned short>(stoul(host_port.substr(host_end+1)));
}
return parsed_host_port;
}
virtual void connect()=0;
std::shared_ptr<boost::asio::deadline_timer> get_timeout_timer(size_t timeout=0) {
if(timeout==0)
timeout=config.timeout;
if(timeout==0)
return nullptr;
auto timer=std::make_shared<boost::asio::deadline_timer>(io_service);
timer->expires_from_now(boost::posix_time::seconds(timeout));
timer->async_wait([this](const boost::system::error_code& ec) {
if(!ec) {
close();
}
});
return timer;
}
void parse_response_header(const std::shared_ptr<Response> &response) const {
std::string line;
getline(response->content, line);
size_t version_end=line.find(' ');
if(version_end!=std::string::npos) {
if(5<line.size())
response->http_version=line.substr(5, version_end-5);
if((version_end+1)<line.size())
response->status_code=line.substr(version_end+1, line.size()-(version_end+1)-1);
getline(response->content, line);
size_t param_end;
while((param_end=line.find(':'))!=std::string::npos) {
size_t value_start=param_end+1;
if((value_start)<line.size()) {
if(line[value_start]==' ')
value_start++;
if(value_start<line.size())
response->header.insert(std::make_pair(line.substr(0, param_end), line.substr(value_start, line.size()-value_start-1)));
}
getline(response->content, line);
}
}
}
std::shared_ptr<Response> request_read() {
std::shared_ptr<Response> response(new Response());
boost::asio::streambuf chunked_streambuf;
auto timer=get_timeout_timer();
boost::asio::async_read_until(*socket, response->content_buffer, "\r\n\r\n",
[this, &response, &chunked_streambuf, timer](const boost::system::error_code& ec, size_t bytes_transferred) {
if(timer)
timer->cancel();
if(!ec) {
size_t num_additional_bytes=response->content_buffer.size()-bytes_transferred;
parse_response_header(response);
auto header_it=response->header.find("Content-Length");
if(header_it!=response->header.end()) {
auto content_length=stoull(header_it->second);
if(content_length>num_additional_bytes) {
auto timer=get_timeout_timer();
boost::asio::async_read(*socket, response->content_buffer,
boost::asio::transfer_exactly(content_length-num_additional_bytes),
[this, timer](const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(ec) {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
}
else if((header_it=response->header.find("Transfer-Encoding"))!=response->header.end() && header_it->second=="chunked") {
request_read_chunked(response, chunked_streambuf);
}
else if(response->http_version<"1.1" || ((header_it=response->header.find("Connection"))!=response->header.end() && header_it->second=="close")) {
auto timer=get_timeout_timer();
boost::asio::async_read(*socket, response->content_buffer,
[this, timer](const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(ec) {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
if(ec!=boost::asio::error::eof)
throw boost::system::system_error(ec);
}
});
}
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
io_service.reset();
io_service.run();
return response;
}
void request_read_chunked(const std::shared_ptr<Response> &response, boost::asio::streambuf &streambuf) {
auto timer=get_timeout_timer();
boost::asio::async_read_until(*socket, response->content_buffer, "\r\n",
[this, &response, &streambuf, timer](const boost::system::error_code& ec, size_t bytes_transferred) {
if(timer)
timer->cancel();
if(!ec) {
std::string line;
getline(response->content, line);
bytes_transferred-=line.size()+1;
line.pop_back();
std::streamsize length=stol(line, 0, 16);
auto num_additional_bytes=static_cast<std::streamsize>(response->content_buffer.size()-bytes_transferred);
auto post_process=[this, &response, &streambuf, length] {
std::ostream stream(&streambuf);
if(length>0) {
std::vector<char> buffer(static_cast<size_t>(length));
response->content.read(&buffer[0], length);
stream.write(&buffer[0], length);
}
//Remove "\r\n"
response->content.get();
response->content.get();
if(length>0)
request_read_chunked(response, streambuf);
else {
std::ostream response_stream(&response->content_buffer);
response_stream << stream.rdbuf();
}
};
if((2+length)>num_additional_bytes) {
auto timer=get_timeout_timer();
boost::asio::async_read(*socket, response->content_buffer,
boost::asio::transfer_exactly(2+length-num_additional_bytes),
[this, post_process, timer](const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(!ec) {
post_process();
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
else
post_process();
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
};
template<class socket_type>
class Client : public ClientBase<socket_type> {};
typedef boost::asio::ip::tcp::socket HTTP;
template<>
class Client<HTTP> : public ClientBase<HTTP> {
public:
Client(const std::string& server_port_path) : ClientBase<HTTP>::ClientBase(server_port_path, 80) {}
protected:
void connect() {
if(!socket || !socket->is_open()) {
std::unique_ptr<boost::asio::ip::tcp::resolver::query> query;
if(config.proxy_server.empty())
query=std::unique_ptr<boost::asio::ip::tcp::resolver::query>(new boost::asio::ip::tcp::resolver::query(host, std::to_string(port)));
else {
auto proxy_host_port=parse_host_port(config.proxy_server, 8080);
query=std::unique_ptr<boost::asio::ip::tcp::resolver::query>(new boost::asio::ip::tcp::resolver::query(proxy_host_port.first, std::to_string(proxy_host_port.second)));
}
resolver.async_resolve(*query, [this](const boost::system::error_code &ec,
boost::asio::ip::tcp::resolver::iterator it){
if(!ec) {
{
std::lock_guard<std::mutex> lock(socket_mutex);
socket=std::unique_ptr<HTTP>(new HTTP(io_service));
}
auto timer=get_timeout_timer(config.timeout_connect);
boost::asio::async_connect(*socket, it, [this, timer]
(const boost::system::error_code &ec, boost::asio::ip::tcp::resolver::iterator /*it*/){
if(timer)
timer->cancel();
if(!ec) {
boost::asio::ip::tcp::no_delay option(true);
this->socket->set_option(option);
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
this->socket=nullptr;
throw boost::system::system_error(ec);
}
});
}
else {
std::lock_guard<std::mutex> lock(socket_mutex);
socket=nullptr;
throw boost::system::system_error(ec);
}
});
io_service.reset();
io_service.run();
}
}
};
}
#endif /* CLIENT_HTTP_HPP */

462
Server/server_http.hpp Executable file
View File

@ -0,0 +1,462 @@
#ifndef SERVER_HTTP_HPP
#define SERVER_HTTP_HPP
#include <boost/asio.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/functional/hash.hpp>
#include <map>
#include <unordered_map>
#include <thread>
#include <functional>
#include <iostream>
#include <sstream>
#include <memory>
#ifndef CASE_INSENSITIVE_EQUALS_AND_HASH
#define CASE_INSENSITIVE_EQUALS_AND_HASH
//Based on http://www.boost.org/doc/libs/1_60_0/doc/html/unordered/hash_equality.html
class case_insensitive_equals {
public:
bool operator()(const std::string &key1, const std::string &key2) const {
return boost::algorithm::iequals(key1, key2);
}
};
class case_insensitive_hash {
public:
size_t operator()(const std::string &key) const {
std::size_t seed=0;
for(auto &c: key)
boost::hash_combine(seed, std::tolower(c));
return seed;
}
};
#endif
// Late 2017 TODO: remove the following checks and always use std::regex
#ifdef USE_BOOST_REGEX
#include <boost/regex.hpp>
#define REGEX_NS boost
#else
#include <regex>
#define REGEX_NS std
#endif
// TODO when switching to c++14, use [[deprecated]] instead
#ifndef DEPRECATED
#ifdef __GNUC__
#define DEPRECATED __attribute__((deprecated))
#elif defined(_MSC_VER)
#define DEPRECATED __declspec(deprecated)
#else
#define DEPRECATED
#endif
#endif
namespace SimpleWeb {
template <class socket_type>
class Server;
template <class socket_type>
class ServerBase {
public:
virtual ~ServerBase() {}
class Response : public std::ostream {
friend class ServerBase<socket_type>;
boost::asio::streambuf streambuf;
std::shared_ptr<socket_type> socket;
Response(const std::shared_ptr<socket_type> &socket): std::ostream(&streambuf), socket(socket) {}
public:
size_t size() {
return streambuf.size();
}
/// If true, force server to close the connection after the response have been sent.
///
/// This is useful when implementing a HTTP/1.0-server sending content
/// without specifying the content length.
bool close_connection_after_response = false;
};
class Content : public std::istream {
friend class ServerBase<socket_type>;
public:
size_t size() {
return streambuf.size();
}
std::string string() {
std::stringstream ss;
ss << rdbuf();
return ss.str();
}
private:
boost::asio::streambuf &streambuf;
Content(boost::asio::streambuf &streambuf): std::istream(&streambuf), streambuf(streambuf) {}
};
class Request {
friend class ServerBase<socket_type>;
friend class Server<socket_type>;
public:
std::string method, path, http_version;
Content content;
std::unordered_multimap<std::string, std::string, case_insensitive_hash, case_insensitive_equals> header;
REGEX_NS::smatch path_match;
std::string remote_endpoint_address;
unsigned short remote_endpoint_port;
private:
Request(const socket_type &socket): content(streambuf) {
try {
remote_endpoint_address=socket.lowest_layer().remote_endpoint().address().to_string();
remote_endpoint_port=socket.lowest_layer().remote_endpoint().port();
}
catch(...) {}
}
boost::asio::streambuf streambuf;
};
class Config {
friend class ServerBase<socket_type>;
Config(unsigned short port): port(port) {}
public:
/// Port number to use. Defaults to 80 for HTTP and 443 for HTTPS.
unsigned short port;
/// Number of threads that the server will use when start() is called. Defaults to 1 thread.
size_t thread_pool_size=1;
/// Timeout on request handling. Defaults to 5 seconds.
size_t timeout_request=5;
/// Timeout on content handling. Defaults to 300 seconds.
size_t timeout_content=300;
/// IPv4 address in dotted decimal form or IPv6 address in hexadecimal notation.
/// If empty, the address will be any address.
std::string address;
/// Set to false to avoid binding the socket to an address that is already in use. Defaults to true.
bool reuse_address=true;
};
///Set before calling start().
Config config;
private:
class regex_orderable : public REGEX_NS::regex {
std::string str;
public:
regex_orderable(const char *regex_cstr) : REGEX_NS::regex(regex_cstr), str(regex_cstr) {}
regex_orderable(const std::string &regex_str) : REGEX_NS::regex(regex_str), str(regex_str) {}
bool operator<(const regex_orderable &rhs) const {
return str<rhs.str;
}
};
public:
/// Warning: do not add or remove resources after start() is called
std::map<regex_orderable, std::map<std::string,
std::function<void(std::shared_ptr<typename ServerBase<socket_type>::Response>, std::shared_ptr<typename ServerBase<socket_type>::Request>)> > > resource;
std::map<std::string,
std::function<void(std::shared_ptr<typename ServerBase<socket_type>::Response>, std::shared_ptr<typename ServerBase<socket_type>::Request>)> > default_resource;
std::function<void(std::shared_ptr<typename ServerBase<socket_type>::Request>, const boost::system::error_code&)> on_error;
std::function<void(std::shared_ptr<socket_type> socket, std::shared_ptr<typename ServerBase<socket_type>::Request>)> on_upgrade;
virtual void start() {
if(!io_service)
io_service=std::make_shared<boost::asio::io_service>();
if(io_service->stopped())
io_service->reset();
boost::asio::ip::tcp::endpoint endpoint;
if(config.address.size()>0)
endpoint=boost::asio::ip::tcp::endpoint(boost::asio::ip::address::from_string(config.address), config.port);
else
endpoint=boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), config.port);
if(!acceptor)
acceptor=std::unique_ptr<boost::asio::ip::tcp::acceptor>(new boost::asio::ip::tcp::acceptor(*io_service));
acceptor->open(endpoint.protocol());
acceptor->set_option(boost::asio::socket_base::reuse_address(config.reuse_address));
acceptor->bind(endpoint);
acceptor->listen();
accept();
//If thread_pool_size>1, start m_io_service.run() in (thread_pool_size-1) threads for thread-pooling
threads.clear();
for(size_t c=1;c<config.thread_pool_size;c++) {
threads.emplace_back([this]() {
io_service->run();
});
}
//Main thread
if(config.thread_pool_size>0)
io_service->run();
//Wait for the rest of the threads, if any, to finish as well
for(auto& t: threads) {
t.join();
}
}
void stop() {
acceptor->close();
if(config.thread_pool_size>0)
io_service->stop();
}
///Use this function if you need to recursively send parts of a longer message
void send(const std::shared_ptr<Response> &response, const std::function<void(const boost::system::error_code&)>& callback=nullptr) const {
boost::asio::async_write(*response->socket, response->streambuf, [this, response, callback](const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(callback)
callback(ec);
});
}
/// If you have your own boost::asio::io_service, store its pointer here before running start().
/// You might also want to set config.thread_pool_size to 0.
std::shared_ptr<boost::asio::io_service> io_service;
protected:
std::unique_ptr<boost::asio::ip::tcp::acceptor> acceptor;
std::vector<std::thread> threads;
ServerBase(unsigned short port) : config(port) {}
virtual void accept()=0;
std::shared_ptr<boost::asio::deadline_timer> get_timeout_timer(const std::shared_ptr<socket_type> &socket, long seconds) {
if(seconds==0)
return nullptr;
auto timer=std::make_shared<boost::asio::deadline_timer>(*io_service);
timer->expires_from_now(boost::posix_time::seconds(seconds));
timer->async_wait([socket](const boost::system::error_code& ec){
if(!ec) {
boost::system::error_code ec;
socket->lowest_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);
socket->lowest_layer().close();
}
});
return timer;
}
void read_request_and_content(const std::shared_ptr<socket_type> &socket) {
//Create new streambuf (Request::streambuf) for async_read_until()
//shared_ptr is used to pass temporary objects to the asynchronous functions
std::shared_ptr<Request> request(new Request(*socket));
//Set timeout on the following boost::asio::async-read or write function
auto timer=this->get_timeout_timer(socket, config.timeout_request);
boost::asio::async_read_until(*socket, request->streambuf, "\r\n\r\n",
[this, socket, request, timer](const boost::system::error_code& ec, size_t bytes_transferred) {
if(timer)
timer->cancel();
if(!ec) {
//request->streambuf.size() is not necessarily the same as bytes_transferred, from Boost-docs:
//"After a successful async_read_until operation, the streambuf may contain additional data beyond the delimiter"
//The chosen solution is to extract lines from the stream directly when parsing the header. What is left of the
//streambuf (maybe some bytes of the content) is appended to in the async_read-function below (for retrieving content).
size_t num_additional_bytes=request->streambuf.size()-bytes_transferred;
if(!this->parse_request(request))
return;
//If content, read that as well
auto it=request->header.find("Content-Length");
if(it!=request->header.end()) {
unsigned long long content_length;
try {
content_length=stoull(it->second);
}
catch(const std::exception &e) {
if(on_error)
on_error(request, boost::system::error_code(boost::system::errc::protocol_error, boost::system::generic_category()));
return;
}
if(content_length>num_additional_bytes) {
//Set timeout on the following boost::asio::async-read or write function
auto timer=this->get_timeout_timer(socket, config.timeout_content);
boost::asio::async_read(*socket, request->streambuf,
boost::asio::transfer_exactly(content_length-num_additional_bytes),
[this, socket, request, timer]
(const boost::system::error_code& ec, size_t /*bytes_transferred*/) {
if(timer)
timer->cancel();
if(!ec)
this->find_resource(socket, request);
else if(on_error)
on_error(request, ec);
});
}
else
this->find_resource(socket, request);
}
else
this->find_resource(socket, request);
}
else if(on_error)
on_error(request, ec);
});
}
bool parse_request(const std::shared_ptr<Request> &request) const {
std::string line;
getline(request->content, line);
size_t method_end;
if((method_end=line.find(' '))!=std::string::npos) {
size_t path_end;
if((path_end=line.find(' ', method_end+1))!=std::string::npos) {
request->method=line.substr(0, method_end);
request->path=line.substr(method_end+1, path_end-method_end-1);
size_t protocol_end;
if((protocol_end=line.find('/', path_end+1))!=std::string::npos) {
if(line.compare(path_end+1, protocol_end-path_end-1, "HTTP")!=0)
return false;
request->http_version=line.substr(protocol_end+1, line.size()-protocol_end-2);
}
else
return false;
getline(request->content, line);
size_t param_end;
while((param_end=line.find(':'))!=std::string::npos) {
size_t value_start=param_end+1;
if((value_start)<line.size()) {
if(line[value_start]==' ')
value_start++;
if(value_start<line.size())
request->header.emplace(line.substr(0, param_end), line.substr(value_start, line.size()-value_start-1));
}
getline(request->content, line);
}
}
else
return false;
}
else
return false;
return true;
}
void find_resource(const std::shared_ptr<socket_type> &socket, const std::shared_ptr<Request> &request) {
//Upgrade connection
if(on_upgrade) {
auto it=request->header.find("Upgrade");
if(it!=request->header.end()) {
on_upgrade(socket, request);
return;
}
}
//Find path- and method-match, and call write_response
for(auto &regex_method: resource) {
auto it=regex_method.second.find(request->method);
if(it!=regex_method.second.end()) {
REGEX_NS::smatch sm_res;
if(REGEX_NS::regex_match(request->path, sm_res, regex_method.first)) {
request->path_match=std::move(sm_res);
write_response(socket, request, it->second);
return;
}
}
}
auto it=default_resource.find(request->method);
if(it!=default_resource.end()) {
write_response(socket, request, it->second);
}
}
void write_response(const std::shared_ptr<socket_type> &socket, const std::shared_ptr<Request> &request,
std::function<void(std::shared_ptr<typename ServerBase<socket_type>::Response>,
std::shared_ptr<typename ServerBase<socket_type>::Request>)>& resource_function) {
//Set timeout on the following boost::asio::async-read or write function
auto timer=this->get_timeout_timer(socket, config.timeout_content);
auto response=std::shared_ptr<Response>(new Response(socket), [this, request, timer](Response *response_ptr) {
auto response=std::shared_ptr<Response>(response_ptr);
this->send(response, [this, response, request, timer](const boost::system::error_code& ec) {
if(timer)
timer->cancel();
if(!ec) {
if (response->close_connection_after_response)
return;
auto range=request->header.equal_range("Connection");
for(auto it=range.first;it!=range.second;it++) {
if(boost::iequals(it->second, "close"))
return;
}
if(request->http_version >= "1.1")
this->read_request_and_content(response->socket);
}
else if(on_error)
on_error(request, ec);
});
});
try {
resource_function(response, request);
}
catch(const std::exception &e) {
if(on_error)
on_error(request, boost::system::error_code(boost::system::errc::operation_canceled, boost::system::generic_category()));
return;
}
}
};
template<class socket_type>
class Server : public ServerBase<socket_type> {};
typedef boost::asio::ip::tcp::socket HTTP;
template<>
class Server<HTTP> : public ServerBase<HTTP> {
public:
DEPRECATED Server(unsigned short port, size_t thread_pool_size=1, long timeout_request=5, long timeout_content=300) :
Server() {
config.port=port;
config.thread_pool_size=thread_pool_size;
config.timeout_request=timeout_request;
config.timeout_content=timeout_content;
}
Server() : ServerBase<HTTP>::ServerBase(80) {}
protected:
void accept() {
//Create new socket for this connection
//Shared_ptr is used to pass temporary objects to the asynchronous functions
auto socket=std::make_shared<HTTP>(*io_service);
acceptor->async_accept(*socket, [this, socket](const boost::system::error_code& ec){
//Immediately start accepting a new connection (if io_service hasn't been stopped)
if (ec != boost::asio::error::operation_aborted)
accept();
if(!ec) {
boost::asio::ip::tcp::no_delay option(true);
socket->set_option(option);
this->read_request_and_content(socket);
}
else if(on_error)
on_error(std::shared_ptr<Request>(new Request(*socket)), ec);
});
}
};
}
#endif /* SERVER_HTTP_HPP */

92
Server/web/admin.html Normal file
View File

@ -0,0 +1,92 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Bootstrap -->
<link rel="stylesheet" href="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
<!--[if lt IE 9]>
<script src="https://cdn.bootcss.com/html5shiv/3.7.3/html5shiv.min.js"></script>
<script src="https://cdn.bootcss.com/respond.js/1.4.2/respond.min.js"></script>
<![endif]-->
</head>
<body>
<div class="container">
<div style="margin: 50px 150px">
<!-- Nav tabs -->
<ul class="nav nav-tabs" role="tablist">
<li role="presentation" class="active"><a href="#build" aria-controls="build" role="tab" data-toggle="tab">build</a></li>
<li role="presentation"><a href="#load" aria-controls="load" role="tab" data-toggle="tab">load</a></li>
<li role="presentation"><a href="#query" aria-controls="query" role="tab" data-toggle="tab">query</a></li>
<li role="presentation"><a href="#unload" aria-controls="unload" role="tab" data-toggle="tab">unload</a></li>
</ul>
<!-- Tab panes -->
<div class="tab-content" style="margin: 20px auto">
<div role="tabpanel" class="tab-pane active" id="build">
<form>
<div class="form-group">
<label for="_db_name">Database name</label>
<input type="text" class="form-control" id="_db_name" name="_db_name" placeholder="db_name">
</div>
<div class="form-group">
<label for="_ds_path">Dataset path</label>
<input type="text" class="form-control" id="_ds_path" name="_ds_path" placeholder="ds_path">
</div>
<button id="build_button" type="button" class="btn btn-default" onclick="build(_db_name.value, _ds_path.value)">build</button>
</form>
</div>
<div role="tabpanel" class="tab-pane" id="load">
<form>
<div class="form-group">
<label for="_db_name_1">Database name</label>
<input type="text" class="form-control" id="_db_name_1" name="_db_name_1" placeholder="db_name">
</div>
<button id="load_button" type="button" class="btn btn-default" onclick="load(_db_name_1.value)">load</button>
</form>
</div>
<div role="tabpanel" class="tab-pane" id="query">
<form>
<div class="form-group">
<label for="query_file">Query file Path</label>
<input type="text" class="form-control" id="query_file" name="query_file" placeholder="query_file_path">
</div>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">query</button>
</form>
<div class="alert alert-warning alert-dismissible fade in" style="margin: 20px auto; height: 400px; overflow: auto" id="queryAns">
</div>
</div>
<div role="tabpanel" class="tab-pane" id="unload">
<form>
<div class="form-group">
<button id="unload_button" type="button" class="btn btn-default" onclick="unload()">unload</button>
</div>
</form>
</div>
</div>
</div>
</div>
</div>
<!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<script src="admin.js"></script>
</body>
</html>

34
Server/web/admin.js Normal file
View File

@ -0,0 +1,34 @@
function build(db, ds){
$.get("build/" + db + "/" + ds, function(data, status){
if(status=="success"){
alert(data);
}
});
}
1
function load(db) {
$.get("load/" + db, function(data, status){
if(status=="success"){
alert(data);
}
});
}
function query(dp) {
$.get("query/" + dp, function(data, status){
if(status=="success"){
$("#queryAns").empty();
var res = $("<p></p>").text(data);
$("#queryAns").append(res);
$("#queryAns").scrollTop($("#queryAns").height());
}
});
}
function unload(db) {
$.get("unload", function(data, status){
if(status=="success"){
alert(data);
}
});
}

77
Server/web/index.html Normal file
View File

@ -0,0 +1,77 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Bootstrap -->
<link rel="stylesheet" href="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
<!--[if lt IE 9]>
<script src="https://cdn.bootcss.com/html5shiv/3.7.3/html5shiv.min.js"></script>
<script src="https://cdn.bootcss.com/respond.js/1.4.2/respond.min.js"></script>
<![endif]-->
</head>
<body>
<div class="container">
<div style="margin: 50px 150px">
<!-- Nav tabs -->
<ul class="nav nav-tabs" role="tablist">
<li role="presentation" class="active"><a href="#load" aria-controls="load" role="tab" data-toggle="tab">load</a></li>
<li role="presentation"><a href="#query" aria-controls="query" role="tab" data-toggle="tab">query</a></li>
<li role="presentation"><a href="#unload" aria-controls="unload" role="tab" data-toggle="tab">unload</a></li>
</ul>
<!-- Tab panes -->
<div class="tab-content" style="margin: 20px auto">
<div role="tabpanel" class="tab-pane active" id="load">
<form>
<div class="form-group">
<label for="_db_name_1">Database name</label>
<input type="text" class="form-control" id="_db_name_1" name="_db_name_1" placeholder="db_name">
</div>
<button id="load_button" type="button" class="btn btn-default" onclick="load(_db_name_1.value)">load</button>
</form>
</div>
<div role="tabpanel" class="tab-pane" id="query">
<form>
<div class="form-group">
<label for="query_file">Query file Path</label>
<input type="text" class="form-control" id="query_file" name="query_file" placeholder="query_file_path">
</div>
<button id="query_button" type="button" class="btn btn-default" onclick="query(query_file.value)">query</button>
</form>
<div class="alert alert-warning alert-dismissible fade in" style="margin: 20px auto; height: 400px; overflow: auto" id="queryAns">
</div>
</div>
<div role="tabpanel" class="tab-pane" id="unload">
<form>
<div class="form-group">
<button id="unload_button" type="button" class="btn btn-default" onclick="unload()">unload</button>
</div>
</form>
</div>
</div>
</div>
</div>
</div>
<!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
<script src="index.js"></script>
</body>
</html>

28
Server/web/index.js Normal file
View File

@ -0,0 +1,28 @@
function load(db) {
//alert("to load"+db);
$.get("load/"+db, function(data, status){
//alert("in get");
if(status=="success"){
alert(data);
}
});
}
function query(dp) {
$.get("query/" + dp, function(data, status){
if(status=="success"){
$("#queryAns").empty();
var res = $("<p></p>").text(data);
$("#queryAns").append(res);
$("#queryAns").scrollTop($("#queryAns").height());
}
});
}
function unload(db) {
$.get("unload", function(data, status){
if(status=="success"){
alert(data);
}
});
}

8
Server/web/test.html Normal file
View File

@ -0,0 +1,8 @@
<html>
<head>
<title>Simple-Web-Server html-file</title>
</head>
<body>
This is the content of test.html
</body>
</html>

8
Util/Util.cpp
View File

@ -1642,3 +1642,11 @@ Util::empty_file(const char* _fname)
}
}
//require that _base>=1
unsigned
ceiling(unsigned _val, unsigned _base)
{
//WARN: we donot check overflow here
return (_val+_base-1) / _base * _base;
}

67
Util/Util.h
View File

@ -266,6 +266,7 @@ public:
static std::string getItemsFromDir(std::string path);
static void logging(std::string _str);
static void empty_file(const char* _fname);
static unsigned ceiling(unsigned _val, unsigned _base);
// Below are some useful hash functions for string
static unsigned simpleHash(const char *_str);
@ -414,5 +415,71 @@ public:
}
};
//NOTICE: bool used to be represented by int in C, but in C++ it only occupies a byte
//But in 32-bit machine, read/write on 32-bit(4-byte) will be more efficient, so bools are compressed into 4-bytes
//vector<bool> is not suggested:)
//http://blog.csdn.net/liushu1231/article/details/8844631
class BoolArray
{
private:
unsigned size;
char* arr;
public:
BoolArray()
{
size = 0;
arr = NULL;
}
BoolArray(unsigned _size)
{
//this->size = (_size+7)/8*8;
this->size = Util::ceiling(_size, 8);
this->arr = new char[this->size/8];
}
void fill(unsigned _size)
{
if(this->arr != NULL)
{
//unsigned tmp = (_size+7)/8*8;
unsigned tmp = Util::ceiling(_size, 8);
if(tmp > this->size)
{
this->size= tmp;
delete[] this->arr;
this->arr = new char[this->size/8];
}
}
else
{
//this->size = (_size+7)/8*8;
this->size = Util::ceiling(_size, 8);
this->arr = new char[this->size/8];
}
}
//void load()
//{
//}
bool exist()
{
return this->size > 0;
}
unsigned getSize()
{
return size;
}
void clear()
{
this->size = 0;
delete[] arr;
arr = NULL;
}
~BoolArray()
{
delete[] arr;
}
};
#endif //_UTIL_UTIL_H

2
init.conf
View File

@ -15,6 +15,8 @@ gstore_mode = single
# please set this option to all and go to modify the debug macros in Util/Util.h (choose to comment out the debug option or not)
debug_level = simple
# TODO: add native/server modes, if in server mode, not output results
[option]
# This option means which directory do you want to place your database in(the directory will be created if not exists)

37
makefile
View File

@ -35,7 +35,7 @@
#compile parameters
CC = ccache g++
CC = ccache g++ -std=c++11
#CC = g++
#the optimazition level of gcc/g++
@ -43,14 +43,15 @@ CC = ccache g++
#NOTICE: -O2 is recommended, while -O3 is dangerous
#when developing, not use -O because it will disturb the normal
#routine. use it for test and release.
CFLAGS = -c -Wall -g -pthread #-fprofile-arcs -ftest-coverage #-pg
EXEFLAG = -g -pthread #-fprofile-arcs -ftest-coverage #-pg
CFLAGS = -c -Wall -O2
EXEFLAG = -O2
#-coverage
#CFLAGS = -c -Wall -O2 -pthread
#EXEFLAG = -O2 -pthread
#add -lreadline -ltermcap if using readline or objs contain readline
library = -ltermcap -lreadline -L./lib -lantlr -lgcov
library = -ltermcap -lreadline -L./lib -L/usr/local/lib -lantlr -lgcov -lboost_filesystem -lboost_system -lboost_regex -lpthread -I/usr/local/include/boost
# library = -ltermcap -lreadline -L./lib -lantlr -lgcov
def64IO = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE
# paths
@ -90,13 +91,16 @@ parserobj = $(objdir)RDFParser.o $(objdir)SparqlParser.o $(objdir)DBparser.o \
serverobj = $(objdir)Operation.o $(objdir)Server.o $(objdir)Client.o $(objdir)Socket.o
# httpobj = $(objdir)client_http.hpp.gch $(objdir)server_http.hpp.gch
databaseobj = $(objdir)Database.o $(objdir)Join.o $(objdir)Strategy.o
objfile = $(kvstoreobj) $(vstreeobj) $(stringindexobj) $(parserobj) $(serverobj) $(databaseobj) \
objfile = $(kvstoreobj) $(vstreeobj) $(stringindexobj) $(parserobj) $(serverobj) $(httpobj) $(databaseobj) \
$(utilobj) $(signatureobj) $(queryobj)
inc = -I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include
#-I./usr/local/include/boost/
#auto generate dependencies
@ -104,7 +108,8 @@ inc = -I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include
# http://blog.csdn.net/jeffrey0000/article/details/12421317
#gtest
TARGET = $(exedir)gbuild $(exedir)gserver $(exedir)gclient $(exedir)gquery $(exedir)gconsole $(api_java) $(exedir)gadd $(exedir)gsub
TARGET = $(exedir)gbuild $(exedir)gserver $(exedir)gclient $(exedir)gquery $(exedir)gconsole $(api_java) $(exedir)gadd $(exedir)gsub $(exedir)HttpConnector
all: $(TARGET)
@ -132,6 +137,10 @@ $(exedir)gclient: $(lib_antlr) $(objdir)gclient.o $(objfile)
$(exedir)gconsole: $(lib_antlr) $(objdir)gconsole.o $(objfile) $(api_cpp)
$(CC) $(EXEFLAG) -o $(exedir)gconsole $(objdir)gconsole.o $(objfile) $(library) -L./api/cpp/lib -lgstoreconnector
$(exedir)HttpConnector: $(lib_antlr) $(objdir)HttpConnector.o ./Server/server_http.hpp ./Server/client_http.hpp $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)HttpConnector $(objdir)HttpConnector.o $(objfile) $(library) $(inc) -DUSE_BOOST_REGEX
#executables end
@ -153,6 +162,10 @@ $(objdir)gclient.o: Main/gclient.cpp Server/Client.h Util/Util.h $(lib_antlr)
$(objdir)gconsole.o: Main/gconsole.cpp Database/Database.h Util/Util.h api/cpp/src/GstoreConnector.h $(lib_antlr)
$(CC) $(CFLAGS) Main/gconsole.cpp $(inc) -o $(objdir)gconsole.o -I./api/cpp/src/ #-DREADLINE_ON
$(objdir)HttpConnector.o: Main/HttpConnector.cpp Server/server_http.hpp Server/client_http.hpp Database/Database.h Util/Util.h $(lib_antlr)
$(CC) $(CFLAGS) Main/HttpConnector.cpp $(inc) -o $(objdir)HttpConnector.o -DUSE_BOOST_REGEX
#objects in Main/ end
@ -390,6 +403,12 @@ $(objdir)Server.o: Server/Server.cpp Server/Server.h $(objdir)Socket.o $(objdir)
$(objdir)Client.o: Server/Client.cpp Server/Client.h $(objdir)Socket.o $(objdir)Util.o
$(CC) $(CFLAGS) Server/Client.cpp $(inc) -o $(objdir)Client.o
# $(objdir)client_http.o: Server/client_http.hpp
# $(CC) $(CFLAGS) Server/client_http.hpp $(inc) -o $(objdir)client_http.o
# $(objdir)server_http.o: Server/server_http.hpp
# $(CC) $(CFLAGS) Server/server_http.hpp $(inc) -o $(objdir)server_http.o
#objects in Server/ end
@ -450,6 +469,12 @@ $(exedir)gadd: $(objdir)gadd.o $(objfile)
$(objdir)gadd.o: Main/gadd.cpp
$(CC) $(CFLAGS) Main/gadd.cpp $(inc) -o $(objdir)gadd.o
#$(objdir)HttpConnector: $(objdir)HttpConnector.o $(objfile)
#$(CC) $(CFLAGS) -o $(exedir)HttpConnector $(objdir)HttpConnector.o $(objfile) lib/libantlr.a $(library) $(inc) -DUSE_BOOST_REGEX
#$(objdir)HttpConnector.o: Main/HttpConnector.cpp
#$(CC) $(CFLAGS) Main/HttpConnector.cpp $(inc) -o $(objdir)HttpConnector.o $(library) -DUSE_BOOST_REGEX
$(exedir)gsub: $(objdir)gsub.o $(objfile)
$(CC) $(EXEFLAG) -o $(exedir)gsub $(objdir)gsub.o $(objfile) lib/libantlr.a $(library)

16
test/run_dbpedia.sh
View File

@ -1,19 +1,19 @@
bin/gload dbpedia.db /home/data/DBpedia/database/dbpedia2014.nt > DBPEDIA/load.txt 2>&1
bin/gbuild dbpedia /home/data/DBpedia/database/dbpedia2014.nt > DBPEDIA/load.txt 2>&1
echo "load finished"
bin/gquery dbpedia.db /home/data/DBpedia/query/q0.sql > DBPEDIA/q0.txt 2>&1
bin/gquery dbpedia /home/data/DBpedia/query/q0.sql > DBPEDIA/q0.txt 2>&1
echo "q0.sql finished"
bin/gquery dbpedia.db /home/data/DBpedia/query/q1.sql > DBPEDIA/q1.txt 2>&1
bin/gquery dbpedia /home/data/DBpedia/query/q1.sql > DBPEDIA/q1.txt 2>&1
echo "q1.sql finished"
bin/gquery dbpedia.db /home/data/DBpedia/query/q2.sql > DBPEDIA/q2.txt 2>&1
bin/gquery dbpedia /home/data/DBpedia/query/q2.sql > DBPEDIA/q2.txt 2>&1
echo "q2.sql finished"
bin/gquery dbpedia.db /home/data/DBpedia/query/q3.sql > DBPEDIA/q3.txt 2>&1
bin/gquery dbpedia /home/data/DBpedia/query/q3.sql > DBPEDIA/q3.txt 2>&1
echo "q3.sql finished"
bin/gquery dbpedia.db /home/data/DBpedia/query/q4.sql > DBPEDIA/q4.txt 2>&1
bin/gquery dbpedia /home/data/DBpedia/query/q4.sql > DBPEDIA/q4.txt 2>&1
echo "q4.sql finished"
bin/gquery dbpedia.db /home/data/DBpedia/query/q5.sql > DBPEDIA/q5.txt 2>&1
bin/gquery dbpedia /home/data/DBpedia/query/q5.sql > DBPEDIA/q5.txt 2>&1
echo "q5.sql finished"
#bin/gquery dbpedia.db /home/data/DBpedia/query/q6.sql > DBPEDIA/q6.txt 2>&1
#bin/gquery dbpedia /home/data/DBpedia/query/q6.sql > DBPEDIA/q6.txt 2>&1
#echo "q6.sql finished"
echo "dbpedia tested"