diff --git a/Main/HttpConnector.cpp b/Main/HttpConnector.cpp new file mode 100755 index 0000000..d38a6ac --- /dev/null +++ b/Main/HttpConnector.cpp @@ -0,0 +1,322 @@ +#include "../Server/server_http.hpp" +#include "../Server/client_http.hpp" + +//Added for the json-example +#define BOOST_SPIRIT_THREADSAFE +#include +#include + +//Added for the default_resource example +#include +#include +//#include +#include +#include +#include + +//db +#include "../Database/Database.h" +#include "../Util/Util.h" + +using namespace std; +//Added for the json-example: +using namespace boost::property_tree; + +typedef SimpleWeb::Server HttpServer; +typedef SimpleWeb::Client HttpClient; + +//Added for the default_resource example +void default_resource_send(const HttpServer &server, const shared_ptr &response, + const shared_ptr &ifs); + +Database *current_database = NULL; + +int main() { + Util util; + //HTTP-server at port 8080 using 1 thread + //Unless you do more heavy non-threaded processing in the resources, + //1 thread is usually faster than several threads + HttpServer server; + server.config.port=8080; + //server.config.port=9000; + //cout<<"after server built"< response, shared_ptr request) { + // server.resource["^/build/([a-zA-Z0-9]+)/([a-zA-Z0-9]+)$"]["GET"]=[&server](shared_ptr response, shared_ptr request) { + server.resource["^/build/([a-zA-Z0-9]*)/(.*)$"]["GET"]=[&server](shared_ptr response, shared_ptr request) { + string db_name=request->path_match[1]; + string db_path=request->path_match[2]; + if(db_name=="" || db_path=="") + { + string error = "Exactly 2 arguments required!"; + // error = db_name + " " + db_path; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + } + + string database = db_name; + if(database.length() > 3 && database.substr(database.length()-3, 3) == ".db") + { + string error = "Your db name to be built should not end with \".db\"."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + } + + //database += ".db"; + string dataset = db_path; + + if(current_database != NULL) + { + string error = "Please unload your database first."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + } + + cout << "Import dataset to build database..." << endl; + cout << "DB_store: " << database << "\tRDF_data: " << dataset << endl; + int len = database.length(); + + current_database = new Database(database); + bool flag = current_database->build(dataset); + delete current_database; + current_database = NULL; + + if(!flag) + { + string error = "Import RDF file to database failed."; + string cmd = "rm -r " + database; + system(cmd.c_str()); + return 0; + } + + // string success = db_name + " " + db_path; + string success = "Import RDF file to database done."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success; + return 0; + }; + + //GET-example for the path /load/[db_name], responds with the matched string in path + //For instance a request GET /load/db123 will receive: db123 + server.resource["^/load/(.*)$"]["GET"]=[&server](shared_ptr response, shared_ptr request) { + string db_name=request->path_match[1]; + + + + if(db_name=="") + { + string error = "Exactly 1 argument is required!"; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + } + + string database = db_name; + if(database.length() > 3 && database.substr(database.length()-3, 3) == ".db") + { + string error = "Your db name to be built should not end with \".db\"."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + } + + //database += ".db"; + + if(current_database != NULL) + { + string error = "Please unload your current database first."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + } + cout << database << endl; + current_database = new Database(database); + bool flag = current_database->load(); + if (!flag) + { + string error = "Failed to load the database."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + delete current_database; + current_database = NULL; + return 0; + } + + + //string success = db_name; + string success = "Database loaded successfully."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success; + + return 0; + }; + + //GET-example for the path /query/[query_file_path], responds with the matched string in path + //For instance a request GET /query/db123 will receive: db123 + server.resource["^/query/(.*)$"]["GET"] = [&server](shared_ptr response, shared_ptr request) { + string db_query=request->path_match[1]; + string str = db_query; + + if(current_database == NULL) + { + string error = "No database in use!"; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + }; + + string sparql; + + if(db_query[0]=='\"') + { + sparql = db_query.substr(1, db_query.length()-2); + } + else + { + string ret = Util::getExactPath(db_query.c_str()); + const char *path = ret.c_str(); + if(path == NULL) + { + string error = "Invalid path of query."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + } + sparql = Util::getQueryFromFile(path); + } + + if (sparql.empty()) { + cerr << "Empty SPARQL." << endl; + return 0; + } + + FILE* output = stdout; + + ResultSet rs; + bool ret = current_database->query(sparql, rs, output); + if(ret) + { + string success = rs.to_str(); + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success; + return 0; + } + else + { + string error = "query() returns false."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + } + }; + + //GET-example for the path /unload/[db_name], responds with the matched string in path + //For instance a request GET /unload/db123 will receive: db123 + server.resource["^/unload$"]["GET"]=[&server](shared_ptr response, shared_ptr request) { + if(current_database == NULL) + { + string error = "No database used now."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error; + return 0; + } + + delete current_database; + current_database = NULL; + string success = "Database unloaded."; + *response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success; + return 0; + }; + + // server.resource["^/json$"]["POST"]=[](shared_ptr response, shared_ptr request) { + // try { + // ptree pt; + // read_json(request->content, pt); + + // string name=pt.get("firstName")+" "+pt.get("lastName"); + + // *response << "HTTP/1.1 200 OK\r\n" + // << "Content-Type: application/json\r\n" + // << "Content-Length: " << name.length() << "\r\n\r\n" + // << name; + // } + // catch(exception& e) { + // *response << "HTTP/1.1 400 Bad Request\r\nContent-Length: " << strlen(e.what()) << "\r\n\r\n" << e.what(); + // } + // }; + + //Default GET-example. If no other matches, this anonymous function will be called. + //Will respond with content in the web/-directory, and its subdirectories. + //Default file: index.html + //Can for instance be used to retrieve an HTML 5 client that uses REST-resources on this server + server.default_resource["GET"]=[&server](shared_ptr response, shared_ptr request) { + try { + auto web_root_path=boost::filesystem::canonical("./Server/web"); + auto path=boost::filesystem::canonical(web_root_path/request->path); + //Check if path is within web_root_path + if(distance(web_root_path.begin(), web_root_path.end())>distance(path.begin(), path.end()) || + !equal(web_root_path.begin(), web_root_path.end(), path.begin())) + throw invalid_argument("path must be within root path"); + if(boost::filesystem::is_directory(path)) + path/="index.html"; + if(!(boost::filesystem::exists(path) && boost::filesystem::is_regular_file(path))) + throw invalid_argument("file does not exist"); + + std::string cache_control, etag; + + // Uncomment the following line to enable Cache-Control + // cache_control="Cache-Control: max-age=86400\r\n"; + + auto ifs=make_shared(); + ifs->open(path.string(), ifstream::in | ios::binary | ios::ate); + + if(*ifs) { + auto length=ifs->tellg(); + ifs->seekg(0, ios::beg); + + *response << "HTTP/1.1 200 OK\r\n" << cache_control << etag << "Content-Length: " << length << "\r\n\r\n"; + default_resource_send(server, response, ifs); + } + else + throw invalid_argument("could not read file"); + } + catch(const exception &e) { + string content="Could not open path "+request->path+": "+e.what(); + *response << "HTTP/1.1 400 Bad Request\r\nContent-Length: " << content.length() << "\r\n\r\n" << content; + } + }; + + thread server_thread([&server](){ + //Start server + server.start(); + }); + + //Wait for server to start so that the client can connect + this_thread::sleep_for(chrono::seconds(1)); + + // //Client examples + // HttpClient client("localhost:8080"); + // auto r1=client.request("GET", "/match/123"); + // cout << r1->content.rdbuf() << endl; + + // string json_string="{\"firstName\": \"John\",\"lastName\": \"Smith\",\"age\": 25}"; + // auto r2=client.request("POST", "/string", json_string); + // cout << r2->content.rdbuf() << endl; + + // auto r3=client.request("POST", "/json", json_string); + // cout << r3->content.rdbuf() << endl; + + server_thread.join(); + + return 0; +} + +void default_resource_send(const HttpServer &server, const shared_ptr &response, + const shared_ptr &ifs) { + //read and send 128 KB at a time + static vector buffer(131072); // Safe when server is running on one thread + streamsize read_length; + if((read_length=ifs->read(&buffer[0], buffer.size()).gcount())>0) { + response->write(&buffer[0], read_length); + if(read_length==static_cast(buffer.size())) { + server.send(response, [&server, response, ifs](const boost::system::error_code &ec) { + if(!ec) + default_resource_send(server, response, ifs); + else + cerr << "Connection interrupted" << endl; + }); + } + } +} + diff --git a/Server/LICENSE b/Server/LICENSE new file mode 100755 index 0000000..7bfd646 --- /dev/null +++ b/Server/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2016 Ole Christian Eidheim + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Server/README.md b/Server/README.md new file mode 100755 index 0000000..7ce6659 --- /dev/null +++ b/Server/README.md @@ -0,0 +1,8 @@ +A Http interface for gStore +================= + +Based on Simple-Web-Server (https://travis-ci.org/eidheim/Simple-Web-Server) + +To see the whole project about gStore, click at https://github.com/Caesar11/gStore + + diff --git a/Server/client_http.hpp b/Server/client_http.hpp new file mode 100755 index 0000000..b77fae7 --- /dev/null +++ b/Server/client_http.hpp @@ -0,0 +1,440 @@ +#ifndef CLIENT_HTTP_HPP +#define CLIENT_HTTP_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifndef CASE_INSENSITIVE_EQUALS_AND_HASH +#define CASE_INSENSITIVE_EQUALS_AND_HASH +//Based on http://www.boost.org/doc/libs/1_60_0/doc/html/unordered/hash_equality.html +class case_insensitive_equals { +public: + bool operator()(const std::string &key1, const std::string &key2) const { + return boost::algorithm::iequals(key1, key2); + } +}; +class case_insensitive_hash { +public: + size_t operator()(const std::string &key) const { + std::size_t seed=0; + for(auto &c: key) + boost::hash_combine(seed, std::tolower(c)); + return seed; + } +}; +#endif + +namespace SimpleWeb { + template + class Client; + + template + class ClientBase { + public: + virtual ~ClientBase() {} + + class Response { + friend class ClientBase; + friend class Client; + public: + std::string http_version, status_code; + + std::istream content; + + std::unordered_multimap header; + + private: + boost::asio::streambuf content_buffer; + + Response(): content(&content_buffer) {} + }; + + class Config { + friend class ClientBase; + private: + Config() {} + public: + /// Set timeout on requests in seconds. Default value: 0 (no timeout). + size_t timeout=0; + /// Set connect timeout in seconds. Default value: 0 (Config::timeout is then used instead). + size_t timeout_connect=0; + /// Set proxy server (server:port) + std::string proxy_server; + }; + + /// Set before calling request + Config config; + + std::shared_ptr request(const std::string& request_type, const std::string& path="/", boost::string_ref content="", + const std::map& header=std::map()) { + auto corrected_path=path; + if(corrected_path=="") + corrected_path="/"; + if(!config.proxy_server.empty() && std::is_same::value) + corrected_path="http://"+host+':'+std::to_string(port)+corrected_path; + + boost::asio::streambuf write_buffer; + std::ostream write_stream(&write_buffer); + write_stream << request_type << " " << corrected_path << " HTTP/1.1\r\n"; + write_stream << "Host: " << host << "\r\n"; + for(auto& h: header) { + write_stream << h.first << ": " << h.second << "\r\n"; + } + if(content.size()>0) + write_stream << "Content-Length: " << content.size() << "\r\n"; + write_stream << "\r\n"; + + connect(); + + auto timer=get_timeout_timer(); + boost::asio::async_write(*socket, write_buffer, + [this, &content, timer](const boost::system::error_code &ec, size_t /*bytes_transferred*/) { + if(timer) + timer->cancel(); + if(!ec) { + if(!content.empty()) { + auto timer=get_timeout_timer(); + boost::asio::async_write(*socket, boost::asio::buffer(content.data(), content.size()), + [this, timer](const boost::system::error_code &ec, size_t /*bytes_transferred*/) { + if(timer) + timer->cancel(); + if(ec) { + std::lock_guard lock(socket_mutex); + this->socket=nullptr; + throw boost::system::system_error(ec); + } + }); + } + } + else { + std::lock_guard lock(socket_mutex); + socket=nullptr; + throw boost::system::system_error(ec); + } + }); + io_service.reset(); + io_service.run(); + + return request_read(); + } + + std::shared_ptr request(const std::string& request_type, const std::string& path, std::iostream& content, + const std::map& header=std::map()) { + auto corrected_path=path; + if(corrected_path=="") + corrected_path="/"; + if(!config.proxy_server.empty() && std::is_same::value) + corrected_path="http://"+host+':'+std::to_string(port)+corrected_path; + + content.seekp(0, std::ios::end); + auto content_length=content.tellp(); + content.seekp(0, std::ios::beg); + + boost::asio::streambuf write_buffer; + std::ostream write_stream(&write_buffer); + write_stream << request_type << " " << corrected_path << " HTTP/1.1\r\n"; + write_stream << "Host: " << host << "\r\n"; + for(auto& h: header) { + write_stream << h.first << ": " << h.second << "\r\n"; + } + if(content_length>0) + write_stream << "Content-Length: " << content_length << "\r\n"; + write_stream << "\r\n"; + if(content_length>0) + write_stream << content.rdbuf(); + + connect(); + + auto timer=get_timeout_timer(); + boost::asio::async_write(*socket, write_buffer, + [this, timer](const boost::system::error_code &ec, size_t /*bytes_transferred*/) { + if(timer) + timer->cancel(); + if(ec) { + std::lock_guard lock(socket_mutex); + socket=nullptr; + throw boost::system::system_error(ec); + } + }); + io_service.reset(); + io_service.run(); + + return request_read(); + } + + void close() { + std::lock_guard lock(socket_mutex); + if(socket) { + boost::system::error_code ec; + socket->lowest_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); + socket->lowest_layer().close(); + } + } + + protected: + boost::asio::io_service io_service; + boost::asio::ip::tcp::resolver resolver; + + std::unique_ptr socket; + std::mutex socket_mutex; + + std::string host; + unsigned short port; + + ClientBase(const std::string& host_port, unsigned short default_port) : resolver(io_service) { + auto parsed_host_port=parse_host_port(host_port, default_port); + host=parsed_host_port.first; + port=parsed_host_port.second; + } + + std::pair parse_host_port(const std::string &host_port, unsigned short default_port) { + std::pair parsed_host_port; + size_t host_end=host_port.find(':'); + if(host_end==std::string::npos) { + parsed_host_port.first=host_port; + parsed_host_port.second=default_port; + } + else { + parsed_host_port.first=host_port.substr(0, host_end); + parsed_host_port.second=static_cast(stoul(host_port.substr(host_end+1))); + } + return parsed_host_port; + } + + virtual void connect()=0; + + std::shared_ptr get_timeout_timer(size_t timeout=0) { + if(timeout==0) + timeout=config.timeout; + if(timeout==0) + return nullptr; + + auto timer=std::make_shared(io_service); + timer->expires_from_now(boost::posix_time::seconds(timeout)); + timer->async_wait([this](const boost::system::error_code& ec) { + if(!ec) { + close(); + } + }); + return timer; + } + + void parse_response_header(const std::shared_ptr &response) const { + std::string line; + getline(response->content, line); + size_t version_end=line.find(' '); + if(version_end!=std::string::npos) { + if(5http_version=line.substr(5, version_end-5); + if((version_end+1)status_code=line.substr(version_end+1, line.size()-(version_end+1)-1); + + getline(response->content, line); + size_t param_end; + while((param_end=line.find(':'))!=std::string::npos) { + size_t value_start=param_end+1; + if((value_start)header.insert(std::make_pair(line.substr(0, param_end), line.substr(value_start, line.size()-value_start-1))); + } + + getline(response->content, line); + } + } + } + + std::shared_ptr request_read() { + std::shared_ptr response(new Response()); + + boost::asio::streambuf chunked_streambuf; + + auto timer=get_timeout_timer(); + boost::asio::async_read_until(*socket, response->content_buffer, "\r\n\r\n", + [this, &response, &chunked_streambuf, timer](const boost::system::error_code& ec, size_t bytes_transferred) { + if(timer) + timer->cancel(); + if(!ec) { + size_t num_additional_bytes=response->content_buffer.size()-bytes_transferred; + + parse_response_header(response); + + auto header_it=response->header.find("Content-Length"); + if(header_it!=response->header.end()) { + auto content_length=stoull(header_it->second); + if(content_length>num_additional_bytes) { + auto timer=get_timeout_timer(); + boost::asio::async_read(*socket, response->content_buffer, + boost::asio::transfer_exactly(content_length-num_additional_bytes), + [this, timer](const boost::system::error_code& ec, size_t /*bytes_transferred*/) { + if(timer) + timer->cancel(); + if(ec) { + std::lock_guard lock(socket_mutex); + this->socket=nullptr; + throw boost::system::system_error(ec); + } + }); + } + } + else if((header_it=response->header.find("Transfer-Encoding"))!=response->header.end() && header_it->second=="chunked") { + request_read_chunked(response, chunked_streambuf); + } + else if(response->http_version<"1.1" || ((header_it=response->header.find("Connection"))!=response->header.end() && header_it->second=="close")) { + auto timer=get_timeout_timer(); + boost::asio::async_read(*socket, response->content_buffer, + [this, timer](const boost::system::error_code& ec, size_t /*bytes_transferred*/) { + if(timer) + timer->cancel(); + if(ec) { + std::lock_guard lock(socket_mutex); + this->socket=nullptr; + if(ec!=boost::asio::error::eof) + throw boost::system::system_error(ec); + } + }); + } + } + else { + std::lock_guard lock(socket_mutex); + socket=nullptr; + throw boost::system::system_error(ec); + } + }); + io_service.reset(); + io_service.run(); + + return response; + } + + void request_read_chunked(const std::shared_ptr &response, boost::asio::streambuf &streambuf) { + auto timer=get_timeout_timer(); + boost::asio::async_read_until(*socket, response->content_buffer, "\r\n", + [this, &response, &streambuf, timer](const boost::system::error_code& ec, size_t bytes_transferred) { + if(timer) + timer->cancel(); + if(!ec) { + std::string line; + getline(response->content, line); + bytes_transferred-=line.size()+1; + line.pop_back(); + std::streamsize length=stol(line, 0, 16); + + auto num_additional_bytes=static_cast(response->content_buffer.size()-bytes_transferred); + + auto post_process=[this, &response, &streambuf, length] { + std::ostream stream(&streambuf); + if(length>0) { + std::vector buffer(static_cast(length)); + response->content.read(&buffer[0], length); + stream.write(&buffer[0], length); + } + + //Remove "\r\n" + response->content.get(); + response->content.get(); + + if(length>0) + request_read_chunked(response, streambuf); + else { + std::ostream response_stream(&response->content_buffer); + response_stream << stream.rdbuf(); + } + }; + + if((2+length)>num_additional_bytes) { + auto timer=get_timeout_timer(); + boost::asio::async_read(*socket, response->content_buffer, + boost::asio::transfer_exactly(2+length-num_additional_bytes), + [this, post_process, timer](const boost::system::error_code& ec, size_t /*bytes_transferred*/) { + if(timer) + timer->cancel(); + if(!ec) { + post_process(); + } + else { + std::lock_guard lock(socket_mutex); + this->socket=nullptr; + throw boost::system::system_error(ec); + } + }); + } + else + post_process(); + } + else { + std::lock_guard lock(socket_mutex); + socket=nullptr; + throw boost::system::system_error(ec); + } + }); + } + }; + + template + class Client : public ClientBase {}; + + typedef boost::asio::ip::tcp::socket HTTP; + + template<> + class Client : public ClientBase { + public: + Client(const std::string& server_port_path) : ClientBase::ClientBase(server_port_path, 80) {} + + protected: + void connect() { + if(!socket || !socket->is_open()) { + std::unique_ptr query; + if(config.proxy_server.empty()) + query=std::unique_ptr(new boost::asio::ip::tcp::resolver::query(host, std::to_string(port))); + else { + auto proxy_host_port=parse_host_port(config.proxy_server, 8080); + query=std::unique_ptr(new boost::asio::ip::tcp::resolver::query(proxy_host_port.first, std::to_string(proxy_host_port.second))); + } + resolver.async_resolve(*query, [this](const boost::system::error_code &ec, + boost::asio::ip::tcp::resolver::iterator it){ + if(!ec) { + { + std::lock_guard lock(socket_mutex); + socket=std::unique_ptr(new HTTP(io_service)); + } + + auto timer=get_timeout_timer(config.timeout_connect); + boost::asio::async_connect(*socket, it, [this, timer] + (const boost::system::error_code &ec, boost::asio::ip::tcp::resolver::iterator /*it*/){ + if(timer) + timer->cancel(); + if(!ec) { + boost::asio::ip::tcp::no_delay option(true); + this->socket->set_option(option); + } + else { + std::lock_guard lock(socket_mutex); + this->socket=nullptr; + throw boost::system::system_error(ec); + } + }); + } + else { + std::lock_guard lock(socket_mutex); + socket=nullptr; + throw boost::system::system_error(ec); + } + }); + io_service.reset(); + io_service.run(); + } + } + }; +} + +#endif /* CLIENT_HTTP_HPP */ diff --git a/Server/server_http.hpp b/Server/server_http.hpp new file mode 100755 index 0000000..2034675 --- /dev/null +++ b/Server/server_http.hpp @@ -0,0 +1,462 @@ +#ifndef SERVER_HTTP_HPP +#define SERVER_HTTP_HPP + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef CASE_INSENSITIVE_EQUALS_AND_HASH +#define CASE_INSENSITIVE_EQUALS_AND_HASH +//Based on http://www.boost.org/doc/libs/1_60_0/doc/html/unordered/hash_equality.html +class case_insensitive_equals { +public: + bool operator()(const std::string &key1, const std::string &key2) const { + return boost::algorithm::iequals(key1, key2); + } +}; +class case_insensitive_hash { +public: + size_t operator()(const std::string &key) const { + std::size_t seed=0; + for(auto &c: key) + boost::hash_combine(seed, std::tolower(c)); + return seed; + } +}; +#endif + +// Late 2017 TODO: remove the following checks and always use std::regex +#ifdef USE_BOOST_REGEX +#include +#define REGEX_NS boost +#else +#include +#define REGEX_NS std +#endif + +// TODO when switching to c++14, use [[deprecated]] instead +#ifndef DEPRECATED +#ifdef __GNUC__ +#define DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +#define DEPRECATED __declspec(deprecated) +#else +#define DEPRECATED +#endif +#endif + +namespace SimpleWeb { + template + class Server; + + template + class ServerBase { + public: + virtual ~ServerBase() {} + + class Response : public std::ostream { + friend class ServerBase; + + boost::asio::streambuf streambuf; + + std::shared_ptr socket; + + Response(const std::shared_ptr &socket): std::ostream(&streambuf), socket(socket) {} + + public: + size_t size() { + return streambuf.size(); + } + + /// If true, force server to close the connection after the response have been sent. + /// + /// This is useful when implementing a HTTP/1.0-server sending content + /// without specifying the content length. + bool close_connection_after_response = false; + }; + + class Content : public std::istream { + friend class ServerBase; + public: + size_t size() { + return streambuf.size(); + } + std::string string() { + std::stringstream ss; + ss << rdbuf(); + return ss.str(); + } + private: + boost::asio::streambuf &streambuf; + Content(boost::asio::streambuf &streambuf): std::istream(&streambuf), streambuf(streambuf) {} + }; + + class Request { + friend class ServerBase; + friend class Server; + public: + std::string method, path, http_version; + + Content content; + + std::unordered_multimap header; + + REGEX_NS::smatch path_match; + + std::string remote_endpoint_address; + unsigned short remote_endpoint_port; + + private: + Request(const socket_type &socket): content(streambuf) { + try { + remote_endpoint_address=socket.lowest_layer().remote_endpoint().address().to_string(); + remote_endpoint_port=socket.lowest_layer().remote_endpoint().port(); + } + catch(...) {} + } + + boost::asio::streambuf streambuf; + }; + + class Config { + friend class ServerBase; + + Config(unsigned short port): port(port) {} + public: + /// Port number to use. Defaults to 80 for HTTP and 443 for HTTPS. + unsigned short port; + /// Number of threads that the server will use when start() is called. Defaults to 1 thread. + size_t thread_pool_size=1; + /// Timeout on request handling. Defaults to 5 seconds. + size_t timeout_request=5; + /// Timeout on content handling. Defaults to 300 seconds. + size_t timeout_content=300; + /// IPv4 address in dotted decimal form or IPv6 address in hexadecimal notation. + /// If empty, the address will be any address. + std::string address; + /// Set to false to avoid binding the socket to an address that is already in use. Defaults to true. + bool reuse_address=true; + }; + ///Set before calling start(). + Config config; + + private: + class regex_orderable : public REGEX_NS::regex { + std::string str; + public: + regex_orderable(const char *regex_cstr) : REGEX_NS::regex(regex_cstr), str(regex_cstr) {} + regex_orderable(const std::string ®ex_str) : REGEX_NS::regex(regex_str), str(regex_str) {} + bool operator<(const regex_orderable &rhs) const { + return str::Response>, std::shared_ptr::Request>)> > > resource; + + std::map::Response>, std::shared_ptr::Request>)> > default_resource; + + std::function::Request>, const boost::system::error_code&)> on_error; + + std::function socket, std::shared_ptr::Request>)> on_upgrade; + + virtual void start() { + if(!io_service) + io_service=std::make_shared(); + + if(io_service->stopped()) + io_service->reset(); + + boost::asio::ip::tcp::endpoint endpoint; + if(config.address.size()>0) + endpoint=boost::asio::ip::tcp::endpoint(boost::asio::ip::address::from_string(config.address), config.port); + else + endpoint=boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), config.port); + + if(!acceptor) + acceptor=std::unique_ptr(new boost::asio::ip::tcp::acceptor(*io_service)); + acceptor->open(endpoint.protocol()); + acceptor->set_option(boost::asio::socket_base::reuse_address(config.reuse_address)); + acceptor->bind(endpoint); + acceptor->listen(); + + accept(); + + //If thread_pool_size>1, start m_io_service.run() in (thread_pool_size-1) threads for thread-pooling + threads.clear(); + for(size_t c=1;crun(); + }); + } + + //Main thread + if(config.thread_pool_size>0) + io_service->run(); + + //Wait for the rest of the threads, if any, to finish as well + for(auto& t: threads) { + t.join(); + } + } + + void stop() { + acceptor->close(); + if(config.thread_pool_size>0) + io_service->stop(); + } + + ///Use this function if you need to recursively send parts of a longer message + void send(const std::shared_ptr &response, const std::function& callback=nullptr) const { + boost::asio::async_write(*response->socket, response->streambuf, [this, response, callback](const boost::system::error_code& ec, size_t /*bytes_transferred*/) { + if(callback) + callback(ec); + }); + } + + /// If you have your own boost::asio::io_service, store its pointer here before running start(). + /// You might also want to set config.thread_pool_size to 0. + std::shared_ptr io_service; + protected: + std::unique_ptr acceptor; + std::vector threads; + + ServerBase(unsigned short port) : config(port) {} + + virtual void accept()=0; + + std::shared_ptr get_timeout_timer(const std::shared_ptr &socket, long seconds) { + if(seconds==0) + return nullptr; + + auto timer=std::make_shared(*io_service); + timer->expires_from_now(boost::posix_time::seconds(seconds)); + timer->async_wait([socket](const boost::system::error_code& ec){ + if(!ec) { + boost::system::error_code ec; + socket->lowest_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); + socket->lowest_layer().close(); + } + }); + return timer; + } + + void read_request_and_content(const std::shared_ptr &socket) { + //Create new streambuf (Request::streambuf) for async_read_until() + //shared_ptr is used to pass temporary objects to the asynchronous functions + std::shared_ptr request(new Request(*socket)); + + //Set timeout on the following boost::asio::async-read or write function + auto timer=this->get_timeout_timer(socket, config.timeout_request); + + boost::asio::async_read_until(*socket, request->streambuf, "\r\n\r\n", + [this, socket, request, timer](const boost::system::error_code& ec, size_t bytes_transferred) { + if(timer) + timer->cancel(); + if(!ec) { + //request->streambuf.size() is not necessarily the same as bytes_transferred, from Boost-docs: + //"After a successful async_read_until operation, the streambuf may contain additional data beyond the delimiter" + //The chosen solution is to extract lines from the stream directly when parsing the header. What is left of the + //streambuf (maybe some bytes of the content) is appended to in the async_read-function below (for retrieving content). + size_t num_additional_bytes=request->streambuf.size()-bytes_transferred; + + if(!this->parse_request(request)) + return; + + //If content, read that as well + auto it=request->header.find("Content-Length"); + if(it!=request->header.end()) { + unsigned long long content_length; + try { + content_length=stoull(it->second); + } + catch(const std::exception &e) { + if(on_error) + on_error(request, boost::system::error_code(boost::system::errc::protocol_error, boost::system::generic_category())); + return; + } + if(content_length>num_additional_bytes) { + //Set timeout on the following boost::asio::async-read or write function + auto timer=this->get_timeout_timer(socket, config.timeout_content); + boost::asio::async_read(*socket, request->streambuf, + boost::asio::transfer_exactly(content_length-num_additional_bytes), + [this, socket, request, timer] + (const boost::system::error_code& ec, size_t /*bytes_transferred*/) { + if(timer) + timer->cancel(); + if(!ec) + this->find_resource(socket, request); + else if(on_error) + on_error(request, ec); + }); + } + else + this->find_resource(socket, request); + } + else + this->find_resource(socket, request); + } + else if(on_error) + on_error(request, ec); + }); + } + + bool parse_request(const std::shared_ptr &request) const { + std::string line; + getline(request->content, line); + size_t method_end; + if((method_end=line.find(' '))!=std::string::npos) { + size_t path_end; + if((path_end=line.find(' ', method_end+1))!=std::string::npos) { + request->method=line.substr(0, method_end); + request->path=line.substr(method_end+1, path_end-method_end-1); + + size_t protocol_end; + if((protocol_end=line.find('/', path_end+1))!=std::string::npos) { + if(line.compare(path_end+1, protocol_end-path_end-1, "HTTP")!=0) + return false; + request->http_version=line.substr(protocol_end+1, line.size()-protocol_end-2); + } + else + return false; + + getline(request->content, line); + size_t param_end; + while((param_end=line.find(':'))!=std::string::npos) { + size_t value_start=param_end+1; + if((value_start)header.emplace(line.substr(0, param_end), line.substr(value_start, line.size()-value_start-1)); + } + + getline(request->content, line); + } + } + else + return false; + } + else + return false; + return true; + } + + void find_resource(const std::shared_ptr &socket, const std::shared_ptr &request) { + //Upgrade connection + if(on_upgrade) { + auto it=request->header.find("Upgrade"); + if(it!=request->header.end()) { + on_upgrade(socket, request); + return; + } + } + //Find path- and method-match, and call write_response + for(auto ®ex_method: resource) { + auto it=regex_method.second.find(request->method); + if(it!=regex_method.second.end()) { + REGEX_NS::smatch sm_res; + if(REGEX_NS::regex_match(request->path, sm_res, regex_method.first)) { + request->path_match=std::move(sm_res); + write_response(socket, request, it->second); + return; + } + } + } + auto it=default_resource.find(request->method); + if(it!=default_resource.end()) { + write_response(socket, request, it->second); + } + } + + void write_response(const std::shared_ptr &socket, const std::shared_ptr &request, + std::function::Response>, + std::shared_ptr::Request>)>& resource_function) { + //Set timeout on the following boost::asio::async-read or write function + auto timer=this->get_timeout_timer(socket, config.timeout_content); + + auto response=std::shared_ptr(new Response(socket), [this, request, timer](Response *response_ptr) { + auto response=std::shared_ptr(response_ptr); + this->send(response, [this, response, request, timer](const boost::system::error_code& ec) { + if(timer) + timer->cancel(); + if(!ec) { + if (response->close_connection_after_response) + return; + + auto range=request->header.equal_range("Connection"); + for(auto it=range.first;it!=range.second;it++) { + if(boost::iequals(it->second, "close")) + return; + } + if(request->http_version >= "1.1") + this->read_request_and_content(response->socket); + } + else if(on_error) + on_error(request, ec); + }); + }); + + try { + resource_function(response, request); + } + catch(const std::exception &e) { + if(on_error) + on_error(request, boost::system::error_code(boost::system::errc::operation_canceled, boost::system::generic_category())); + return; + } + } + }; + + template + class Server : public ServerBase {}; + + typedef boost::asio::ip::tcp::socket HTTP; + + template<> + class Server : public ServerBase { + public: + DEPRECATED Server(unsigned short port, size_t thread_pool_size=1, long timeout_request=5, long timeout_content=300) : + Server() { + config.port=port; + config.thread_pool_size=thread_pool_size; + config.timeout_request=timeout_request; + config.timeout_content=timeout_content; + } + + Server() : ServerBase::ServerBase(80) {} + + protected: + void accept() { + //Create new socket for this connection + //Shared_ptr is used to pass temporary objects to the asynchronous functions + auto socket=std::make_shared(*io_service); + + acceptor->async_accept(*socket, [this, socket](const boost::system::error_code& ec){ + //Immediately start accepting a new connection (if io_service hasn't been stopped) + if (ec != boost::asio::error::operation_aborted) + accept(); + + if(!ec) { + boost::asio::ip::tcp::no_delay option(true); + socket->set_option(option); + + this->read_request_and_content(socket); + } + else if(on_error) + on_error(std::shared_ptr(new Request(*socket)), ec); + }); + } + }; +} +#endif /* SERVER_HTTP_HPP */ diff --git a/Server/web/admin.html b/Server/web/admin.html new file mode 100644 index 0000000..a54319b --- /dev/null +++ b/Server/web/admin.html @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + +
+
+ + + + +
+
+
+
+ + +
+
+ + +
+ +
+
+
+
+
+ + +
+ +
+
+
+
+
+ + +
+ +
+
+ +
+
+
+
+
+ +
+
+
+
+ + + + + + + + + +
+
+ + + + + + + + + diff --git a/Server/web/admin.js b/Server/web/admin.js new file mode 100644 index 0000000..7205146 --- /dev/null +++ b/Server/web/admin.js @@ -0,0 +1,34 @@ +function build(db, ds){ + $.get("build/" + db + "/" + ds, function(data, status){ + if(status=="success"){ + alert(data); + } + }); +} + 1 +function load(db) { + $.get("load/" + db, function(data, status){ + if(status=="success"){ + alert(data); + } + }); +} + +function query(dp) { + $.get("query/" + dp, function(data, status){ + if(status=="success"){ + $("#queryAns").empty(); + var res = $("

").text(data); + $("#queryAns").append(res); + $("#queryAns").scrollTop($("#queryAns").height()); + } + }); +} + +function unload(db) { + $.get("unload", function(data, status){ + if(status=="success"){ + alert(data); + } + }); +} diff --git a/Server/web/index.html b/Server/web/index.html new file mode 100644 index 0000000..b47aa3e --- /dev/null +++ b/Server/web/index.html @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + +
+
+ + + + +
+
+
+
+ + +
+ +
+
+
+
+
+ + +
+ +
+
+
+
+
+
+
+ +
+
+
+
+ + + + + + + + + +
+
+ + + + + + + + + diff --git a/Server/web/index.js b/Server/web/index.js new file mode 100644 index 0000000..a6703ed --- /dev/null +++ b/Server/web/index.js @@ -0,0 +1,28 @@ +function load(db) { + //alert("to load"+db); + $.get("load/"+db, function(data, status){ + //alert("in get"); + if(status=="success"){ + alert(data); + } + }); +} + +function query(dp) { + $.get("query/" + dp, function(data, status){ + if(status=="success"){ + $("#queryAns").empty(); + var res = $("

").text(data); + $("#queryAns").append(res); + $("#queryAns").scrollTop($("#queryAns").height()); + } + }); +} + +function unload(db) { + $.get("unload", function(data, status){ + if(status=="success"){ + alert(data); + } + }); +} diff --git a/Server/web/test.html b/Server/web/test.html new file mode 100644 index 0000000..af5fe1c --- /dev/null +++ b/Server/web/test.html @@ -0,0 +1,8 @@ + + + Simple-Web-Server html-file + + + This is the content of test.html + + diff --git a/init.conf b/init.conf index da57dea..c0a12b3 100644 --- a/init.conf +++ b/init.conf @@ -15,6 +15,8 @@ gstore_mode = single # please set this option to all and go to modify the debug macros in Util/Util.h (choose to comment out the debug option or not) debug_level = simple +# TODO: add native/server modes, if in server mode, not output results + [option] # This option means which directory do you want to place your database in(the directory will be created if not exists) diff --git a/makefile b/makefile index 0df68c5..10d10b4 100644 --- a/makefile +++ b/makefile @@ -23,10 +23,6 @@ #http://blog.csdn.net/cscrazybing/article/details/50789482 #http://blog.163.com/liuhonggaono1@126/blog/static/10497901201210254622141/ -#NOTICE: to debug the program, gdb and valgrind can be used -# objdump, nm and size command -#To analyse the performance, gprof and gcov/lcov can be used - #TODO:the dependences are not complete! @@ -47,15 +43,15 @@ CC = ccache g++ -std=c++11 #NOTICE: -O2 is recommended, while -O3 is dangerous #when developing, not use -O because it will disturb the normal #routine. use it for test and release. -CFLAGS = -c -Wall -g -pthread #-fprofile-arcs -ftest-coverage #-pg -EXEFLAG = -g -pthread #-fprofile-arcs -ftest-coverage #-pg +CFLAGS = -c -Wall -O2 +EXEFLAG = -O2 #-coverage #CFLAGS = -c -Wall -O2 -pthread #EXEFLAG = -O2 -pthread #add -lreadline -ltermcap if using readline or objs contain readline -library = -ltermcap -lreadline -L./lib -lantlr -lgcov -library = -ltermcap -lreadline -L./lib -L/usr/local/lib/ -lantlr -lgcov -lboost_filesystem -lboost_system -lpthread +library = -ltermcap -lreadline -L./lib -lantlr -lgcov -lboost_filesystem -lboost_system -lpthread -I/usr/local/include/boost +# library = -ltermcap -lreadline -L./lib -lantlr -lgcov def64IO = -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE # paths @@ -92,15 +88,18 @@ stringindexobj = $(objdir)StringIndex.o parserobj = $(objdir)RDFParser.o $(objdir)SparqlParser.o $(objdir)DBparser.o \ $(objdir)SparqlLexer.o $(objdir)TurtleParser.o $(objdir)QueryParser.o -serverobj = $(objdir)Operation.o $(objdir)Server.o $(objdir)Client.o $(objdir)Socket.o +serverobj = $(objdir)Operation.o $(objdir)Server.o $(objdir)Client.o $(objdir)Socket.o + +# httpobj = $(objdir)client_http.hpp.gch $(objdir)server_http.hpp.gch databaseobj = $(objdir)Database.o $(objdir)Join.o $(objdir)Strategy.o -objfile = $(kvstoreobj) $(vstreeobj) $(stringindexobj) $(parserobj) $(serverobj) $(databaseobj) \ +objfile = $(kvstoreobj) $(vstreeobj) $(stringindexobj) $(parserobj) $(serverobj) $(httpobj) $(databaseobj) \ $(utilobj) $(signatureobj) $(queryobj) -inc = -I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include +inc = -I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include +#-I./usr/local/include/boost/ #auto generate dependencies @@ -108,7 +107,8 @@ inc = -I./tools/libantlr3c-3.4/ -I./tools/libantlr3c-3.4/include # http://blog.csdn.net/jeffrey0000/article/details/12421317 #gtest -TARGET = $(exedir)gbuild $(exedir)gserver $(exedir)gclient $(exedir)gquery $(exedir)gconsole $(api_java) $(exedir)gadd $(exedir)gsub + +TARGET = $(exedir)gbuild $(exedir)gserver $(exedir)gclient $(exedir)gquery $(exedir)gconsole $(api_java) $(exedir)gadd $(exedir)gsub $(exedir)HttpConnector all: $(TARGET) @@ -136,6 +136,10 @@ $(exedir)gclient: $(lib_antlr) $(objdir)gclient.o $(objfile) $(exedir)gconsole: $(lib_antlr) $(objdir)gconsole.o $(objfile) $(api_cpp) $(CC) $(EXEFLAG) -o $(exedir)gconsole $(objdir)gconsole.o $(objfile) $(library) -L./api/cpp/lib -lgstoreconnector +$(exedir)HttpConnector: $(lib_antlr) $(objdir)HttpConnector.o ./Server/server_http.hpp ./Server/client_http.hpp $(objfile) + $(CC) $(EXEFLAG) -o $(exedir)HttpConnector $(objdir)HttpConnector.o $(objfile) $(library) $(inc) + + #executables end @@ -157,6 +161,10 @@ $(objdir)gclient.o: Main/gclient.cpp Server/Client.h Util/Util.h $(lib_antlr) $(objdir)gconsole.o: Main/gconsole.cpp Database/Database.h Util/Util.h api/cpp/src/GstoreConnector.h $(lib_antlr) $(CC) $(CFLAGS) Main/gconsole.cpp $(inc) -o $(objdir)gconsole.o -I./api/cpp/src/ #-DREADLINE_ON +$(objdir)HttpConnector.o: Main/HttpConnector.cpp Server/server_http.hpp Server/client_http.hpp Database/Database.h Util/Util.h $(lib_antlr) + $(CC) $(CFLAGS) Main/HttpConnector.cpp $(inc) -o $(objdir)HttpConnector.o + + #objects in Main/ end @@ -371,6 +379,12 @@ $(objdir)Server.o: Server/Server.cpp Server/Server.h $(objdir)Socket.o $(objdir) $(objdir)Client.o: Server/Client.cpp Server/Client.h $(objdir)Socket.o $(objdir)Util.o $(CC) $(CFLAGS) Server/Client.cpp $(inc) -o $(objdir)Client.o +# $(objdir)client_http.o: Server/client_http.hpp +# $(CC) $(CFLAGS) Server/client_http.hpp $(inc) -o $(objdir)client_http.o + +# $(objdir)server_http.o: Server/server_http.hpp +# $(CC) $(CFLAGS) Server/server_http.hpp $(inc) -o $(objdir)server_http.o + #objects in Server/ end @@ -431,6 +445,12 @@ $(exedir)gadd: $(objdir)gadd.o $(objfile) $(objdir)gadd.o: Main/gadd.cpp $(CC) $(CFLAGS) Main/gadd.cpp $(inc) -o $(objdir)gadd.o +$(objdir)HttpConnector: $(objdir)HttpConnector.o $(objfile) + $(CC) $(CFLAGS) -o $(exedir)HttpConnector $(objdir)HttpConnector.o $(objfile) lib/libantlr.a $(library) $(inc) + +$(objdir)HttpConnector.o: Main/HttpConnector.cpp + $(CC) $(CFLAGS) Main/HttpConnector.cpp $(inc) -o $(objdir)HttpConnector.o $(library) + $(exedir)gsub: $(objdir)gsub.o $(objfile) $(CC) $(EXEFLAG) -o $(exedir)gsub $(objdir)gsub.o $(objfile) lib/libantlr.a $(library)