refactor: fix ghttp and add function

Database/Database.cpp

@@ -300,6 +300,66 @@ Database::writeIDinfo()
 	fp = NULL;
 }
 
+void
+Database::saveIDinfo()
+{
+	//cout<<"now to write the id info"<<endl;
+	FILE* fp = NULL;
+	BlockInfo *bp = NULL, *tp = NULL;
+
+	fp = fopen(this->free_id_file_entity.c_str(), "w+");
+	if (fp == NULL)
+	{
+		cout << "write entity id info error" << endl;
+		return;
+	}
+	fwrite(&(this->limitID_entity), sizeof(int), 1, fp);
+	bp = this->freelist_entity;
+	while (bp != NULL)
+	{
+		fwrite(&(bp->num), sizeof(int), 1, fp);
+		tp = bp->next;
+		bp = tp;
+	}
+	fclose(fp);
+	fp = NULL;
+
+	fp = fopen(this->free_id_file_literal.c_str(), "w+");
+	if (fp == NULL)
+	{
+		cout << "write literal id info error" << endl;
+		return;
+	}
+	fwrite(&(this->limitID_literal), sizeof(int), 1, fp);
+	bp = this->freelist_literal;
+	while (bp != NULL)
+	{
+		fwrite(&(bp->num), sizeof(int), 1, fp);
+		tp = bp->next;
+		bp = tp;
+	}
+	fclose(fp);
+	fp = NULL;
+
+	fp = fopen(this->free_id_file_predicate.c_str(), "w+");
+	if (fp == NULL)
+	{
+		cout << "write predicate id info error" << endl;
+		return;
+	}
+	fwrite(&(this->limitID_predicate), sizeof(int), 1, fp);
+	bp = this->freelist_predicate;
+	while (bp != NULL)
+	{
+		fwrite(&(bp->num), sizeof(int), 1, fp);
+		tp = bp->next;
+		bp = tp;
+	}
+	fclose(fp);
+	fp = NULL;
+}
+
+
 //ID alloc garbage error(LITERAL_FIRST_ID or double) add base for literal
 TYPE_ENTITY_LITERAL_ID
 Database::allocEntityID()
@@ -460,6 +520,7 @@ Database::~Database()
 	//Util::debug_database = NULL;	//debug: when multiple databases
 }
 
+//TODO: update pre map if insert/delete
 void
 Database::setPreMap()
 {
@@ -1112,6 +1173,26 @@ Database::unload()
 	return true;
 }
 
+//this is used for checkpoint, we must ensure that modification is written to disk,
+//so flush() is a must
+bool Database::save()
+{
+	//this->vstree->saveTree();
+	this->kvstore->flush();
+	this->saveDBInfoFile();
+	this->saveIDinfo();
+
+	//TODO: fsync or using sync in Util
+	//should sync every file modified
+	//TODO: add flush for string index
+	//this->stringindex->flush();
+	this->clear_update_log();
+
+	cerr<<"database checkpoint: "<<this->getName()<<endl;
+
+	return true;
+}
+
 void Database::clear() 
 {
 	delete[] this->pre2num;
@@ -1436,6 +1517,8 @@ Database::saveDBInfoFile()
 	fwrite(&this->pre_num, sizeof(int), 1, filePtr);
 	fwrite(&this->literal_num, sizeof(int), 1, filePtr);
 	fwrite(&this->encode_mode, sizeof(int), 1, filePtr);
+
+	fflush(filePtr);
 	fclose(filePtr);
 
 	//Util::triple_num = this->triples_num;

Database/Database.h

@@ -46,6 +46,7 @@ public:
 	void release(FILE* fp0);
 	~Database();
 
+	bool save();
 	bool load();
 	bool unload();
 	void clear();
@@ -200,6 +201,7 @@ private:
 	void resetIDinfo(); //reset the id info for build
 	void readIDinfo();  //read and build the free list
 	void writeIDinfo(); //write and empty the free list
+	void saveIDinfo(); //write and empty the free list
 
 	bool saveDBInfoFile();
 	bool loadDBInfoFile();

KVstore/ISTree/storage/ISStorage.cpp

@@ -562,7 +562,10 @@ ISStorage::writeTree(ISNode* _root)	//write the whole tree back and close treefp
 		fputc(c & ~(1 << j), treefp);
 		bp = bp->next;
 	}
+
+	fflush(this->treefp);
 	//fclose(this->treefp);
+
 	return true;
 }
 

KVstore/IVTree/storage/IVStorage.cpp

@@ -612,7 +612,10 @@ IVStorage::writeTree(IVNode* _root)	//write the whole tree back and close treefp
 		fputc(c & ~(1 << j), treefp);
 		bp = bp->next;
 	}
+
+	fflush(this->treefp);
 	//fclose(this->treefp);
+
 	return true;
 }
 

KVstore/SITree/storage/SIStorage.cpp

@@ -562,6 +562,8 @@ SIStorage::writeTree(SINode* _root)	//write the whole tree back and close treefp
 		fputc(c & ~(1 << j), treefp);
 		bp = bp->next;
 	}
+
+	fflush(this->treefp);
 	//fclose(this->treefp);
 
 	return true;

Main/ghttp.cpp

@@ -10,10 +10,28 @@
 //operation.log: not used
 //query.log: query string, result num, and time of answering
 
+//TODO: to add db_name to all URLs, and change the index.js using /show to get name
+//TODO: modify gmonitor.java
+
 //TODO: mutiple threads , multiple users and multiple databases
 //How to acquire http connection ID? getSocket()  or use username to login?
 //But when setting up endpoint, username may not be a good choice
 
+//TODO: use response<< directly for query result, but for results too large to be placed in memory,
+//file donwload and Stream should be considered, split-page getNext() as well
+//Notice that read each block from disk and transfer to network may be very slow
+//
+//BETTER: define a format to return all information (header and body)
+//
+//TODO: backup() only preserve one version? is remove() and copy() updated to disk in time?
+//how to avoid disk error itself? send to otherwhere?
+//
+//TODO+DEBUG?: write log in update_log in Database.cpp 
+//is ostream.write() ok to update to disk at once? all add ofstream.flush()?
+//Also the checkpoint function!!!
+//http://bookug.cc/rwbuffer
+//BETTER: add a sync function in Util to support FILE*, fd, and fstream
+
 #include "../Server/server_http.hpp"
 #include "../Server/client_http.hpp"
 //db
@@ -309,7 +327,8 @@ int initialize(int argc, char *argv[])
     //     }
     // };
 
-    server.resource["^/%3[F|f]operation%3[D|d]checkpoint%26db_name%3[D|d](.*)$"]["GET"]=[&server](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) 
+	//TODO: use db_name if multiple databases
+    server.resource["^/%3[F|f]operation%3[D|d]checkpoint$"]["GET"]=[&server](shared_ptr<HttpServer::Response> response, shared_ptr<HttpServer::Request> request) 
 	{
 		checkpoint_handler(server, response, request);
     };
@@ -463,10 +482,16 @@ void download_result(const HttpServer& server, const shared_ptr<HttpServer::Resp
 
 				*response << "HTTP/1.1 200 OK\r\n" << cache_control << etag << "Content-Length: " << length << "\r\n\r\n";
 				default_resource_send(server, response, ifs);
+				//DEBUG: in fact this make no sense, result will be received by clients even if server not wait for the network transfer!  (but very slow if results are large)
+				//sleep(10);  //seconds
+				//cout<<"sleep ends"<<endl;
+				//DEBUG: this will close the stream, and the info received by client may be incomplete
+				//ifs->close();
 			}
 			else
 				throw invalid_argument("could not read file.");
 		}
+
 		if((boost::filesystem::exists(path) && boost::filesystem::is_regular_file(path)))
 		{
 			//delete file in delpath.
@@ -823,6 +848,8 @@ bool delete_handler(const HttpServer& server, const shared_ptr<HttpServer::Respo
 	filepath = UrlDecode(filepath);
 
 	download_result(server, response, request, download, filepath);
+	//sleep(1);  //seconds
+	//cout<<"sleep ends"<<endl;
 
 	return true;
 }
@@ -873,17 +900,44 @@ bool default_handler(const HttpServer& server, const shared_ptr<HttpServer::Resp
 	return true;
 }
 
-//BETTER: server choose to save a database when system is not busy
+//TODO+BETTER: server choose to save a database when system is not busy
+//If user send this command too frequently, the performance may be awful if updates are large
 bool checkpoint_handler(const HttpServer& server, const shared_ptr<HttpServer::Response>& response, const shared_ptr<HttpServer::Request>& request)
 {
-	//TODO
+	if(current_database == NULL)
+	{
+		string error = "No database used.";
+		*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
+		return false;
+	}
+
+	//For database checkpoint or log/transaction:
+	//http://www.newsmth.net/nForum/#!article/LinuxDev/37802?p=2
+	//http://blog.csdn.net/cywosp/article/details/8767327
+	//http://www.fx114.net/qa-63-143449.aspx
+	current_database->save();
+	//NOTICE: this info is in header
+	string success = "Database saveed successfully.";
+	//header and content are split by an empty line
+	*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success;
+
 	return true;
 }
 
 //BETTER+TODO: indicate the db_name when query
 bool show_handler(const HttpServer& server, const shared_ptr<HttpServer::Response>& response, const shared_ptr<HttpServer::Request>& request)
 {
-	//TODO
+	if(current_database == NULL)
+	{
+		string error = "No database used.";
+		*response << "HTTP/1.1 200 OK\r\nContent-Length: " << error.length() << "\r\n\r\n" << error;
+		return false;
+	}
+
+	//NOTICE: this info is in header
+	string success = current_database->getName();
+	*response << "HTTP/1.1 200 OK\r\nContent-Length: " << success.length() << "\r\n\r\n" << success;
+
 	return true;
 }
 

api/http/java/example/JavaAPIExample.java

@@ -14,6 +14,13 @@ public class JavaAPIExample
 	{
 		// initialize the GStore server's IP address and port.
 		GstoreConnector gc = new GstoreConnector("127.0.0.1", 9000);
+		//for sparql endpoint, URL can also be used here, like freebase.gstore-pku.com:80
+		//GstoreConnector gc = new GstoreConnector("tourist.gstore-pku.com", 80);
+
+		//test download function
+		//String file = gc.test_download("big.txt");
+		//String file = gc.test_download("big.txt");
+		//System.out.println(file.substring(file.length()-100, file.length()));
 		
 	    // build a new database by a RDF file.
 	    // note that the relative path is related to gserver.

api/http/java/example/Makefile

@@ -1,4 +1,4 @@
-JavaAPIExample.class:
+JavaAPIExample.class: JavaAPIExample.java
 	javac -cp ../lib/GstoreJavaAPI.jar JavaAPIExample.java
 
 .PHONY: clean run

api/http/java/src/Makefile

@@ -6,7 +6,7 @@ all: $(lib_dir)GstoreJavaAPI.jar
 $(lib_dir)GstoreJavaAPI.jar: jgsc/GstoreConnector.class
 	jar -cvf $(lib_dir)GstoreJavaAPI.jar jgsc/GstoreConnector.java jgsc/GstoreConnector.class
 
-jgsc/GstoreConnector.class:
+jgsc/GstoreConnector.class: jgsc/GstoreConnector.java
 	javac jgsc/GstoreConnector.java 
 
 clean:

api/http/java/src/jgsc/GstoreConnector.java

@@ -32,7 +32,7 @@ public class GstoreConnector {
         this.serverPort = _port;
     }
 
-	//TODO: what if the query result is too large?  
+	//TODO: what if the query result is too large?  split and save to file, or use Stream
 	//how about get next (need to record the connection or user in db server)
     public String sendGet(String param) {
 		String url = "http://" + this.serverIP + ":" + this.serverPort;
@@ -66,6 +66,7 @@ public class GstoreConnector {
             for (String key : map.keySet()) {
                 System.out.println(key + "--->" + map.get(key));
             }
+                //System.out.println("============================================");
             // 定义 BufferedReader输入流来读取URL的响应
             in = new BufferedReader(new InputStreamReader(
                     connection.getInputStream()));
@@ -189,6 +190,7 @@ public class GstoreConnector {
 		//catch (UnsupportedEncodingException ex) {
 			//throw new RuntimeException("Broken VM does not support UTF-8");
 		//}
+
 		String cmd = "?operation=query&format=json&sparql=" + _sparql;
         //String cmd = "query/\"" + _sparql + "\"";
         String msg = this.sendGet(cmd);
@@ -223,6 +225,23 @@ public class GstoreConnector {
         return msg;
     }
 
+	public String test_download(String filepath)
+	{
+        boolean connect_return = this.connect();
+        if (!connect_return) {
+            System.err.println("connect to server error. @GstoreConnector.query");
+            return "connect to server error.";
+        }
+
+		//TEST: a small file, a large file
+		String cmd = "?operation=delete&download=true&filepath=" + filepath;
+        String msg = this.sendGet(cmd);
+
+        this.disconnect();
+
+        return msg;
+	}
+
     private boolean connect() {
 		return true;
     }

init.conf

@@ -23,7 +23,7 @@ debug_level = simple
 
 [option]
 
-# This option means which directory do you want to place your database in(the directory will be created if not exists)
+# This option means which directory do you want to place your database in
 # NOTICE:the position is the root of gStore system directory by default
 # db_home = .
  #db_home = /home/ssd

makefile

@@ -43,11 +43,11 @@ CC = g++
 #NOTICE: -O2 is recommended, while -O3 is dangerous
 #when developing, not use -O because it will disturb the normal 
 #routine. use it for test and release.
-#CFLAGS = -c -Wall -O2 -pthread -std=c++11
-#EXEFLAG = -O2 -pthread -std=c++11
+CFLAGS = -c -Wall -O2 -pthread -std=c++11
+EXEFLAG = -O2 -pthread -std=c++11
 #-coverage
-CFLAGS = -c -Wall -pthread -g -std=c++11
-EXEFLAG = -pthread -g -std=c++11
+#CFLAGS = -c -Wall -pthread -g -std=c++11
+#EXEFLAG = -pthread -g -std=c++11
 
 #add -lreadline -ltermcap if using readline or objs contain readline
 library = -ltermcap -lreadline -L./lib -L/usr/local/lib -lantlr -lgcov -lboost_filesystem -lboost_system -lboost_regex -lpthread -I/usr/local/include/boost