with size and targetdir

client.cpp

@@ -1,9 +1,9 @@
 #include "client.h"
-void transfer(const char *action,const char *host,const char *filepath,const char *uid,const char *gid,const char *position)
+void transfer(const char *action,const char *host,const char *filepath,const char *targetdir,const char *uid,const char *gid,const char *position,const char *size)
 {
  
  Py_Initialize();
- if(Py_IsInitialized())
+ if(!Py_IsInitialized())
  {
    printf("can't initialize\n");
  }
@@ -14,17 +14,19 @@ void transfer(const char *action,const char *host,const char *filepath,const cha
  PyRun_SimpleString("import os");
  PyRun_SimpleString("import string");
  PyRun_SimpleString("sys.path.append('./')");
- PyObject *pArgs = PyTuple_New(6);   
+ PyObject *pArgs = PyTuple_New(8);   
  PyTuple_SetItem(pArgs,0,Py_BuildValue("s",action));  
  PyTuple_SetItem(pArgs,1,Py_BuildValue("s",host));  
  PyTuple_SetItem(pArgs,2,Py_BuildValue("s",filepath));
- PyTuple_SetItem(pArgs,3,Py_BuildValue("s",uid));  
- PyTuple_SetItem(pArgs,4,Py_BuildValue("s",gid));  
- PyTuple_SetItem(pArgs,5,Py_BuildValue("s",position));  
+ PyTuple_SetItem(pArgs,3,Py_BuildValue("s",targetdir));
+ PyTuple_SetItem(pArgs,4,Py_BuildValue("s",uid));  
+ PyTuple_SetItem(pArgs,5,Py_BuildValue("s",gid));  
+ PyTuple_SetItem(pArgs,6,Py_BuildValue("s",position));  
+ PyTuple_SetItem(pArgs,7,Py_BuildValue("s",size));  
  pModule = PyImport_ImportModule("client");
  if(pModule == NULL)
  {
-   printf("can't python file\n");
+   printf("can't load module\n");
  }
  pFunc = PyObject_GetAttrString(pModule,"entrance");
  if(pFunc == NULL)

client.h

@@ -3,5 +3,5 @@
 #include <iostream>
 #include <Python.h>
 using namespace std;
-void transfer(const char *action,const char *host,const char *filepath,const char *uid,const char *gid,const char *position);
+void transfer(const char *action,const char *host,const char *filepath,const char *targetdir,const char *uid,const char *gid,const char *position,const char *size);
 #endif

client.py

@@ -1,9 +1,12 @@
 from tornado.httpclient import HTTPRequest, AsyncHTTPClient
+from tornado.simple_httpclient import SimpleAsyncHTTPClient
 import tornado.ioloop
 import tornado.web
 import os,sys
 from tornado import gen
 from functools import partial
+#import time,datatime
+
 
 total_downloaded = 0
 #action = sys.argv[1]
@@ -11,12 +14,12 @@ total_downloaded = 0
 #uid = sys.argv[3]
 #gid = sys.argv[4]
 #pos = sys.argv[5]
-def geturl(action,host,filepath,uid,gid,pos):
+def geturl(action,host,filepath,uid,gid,pos,size):
     if action=="read":
  #       uid = sys.argv[3]
   #      gid = sys.argv[4]
    #     pos = sys.argv[5]
-        url = "http://"+host+":8880/read?filepath="+filepath+"&uid="+uid+"&gid="+gid+"&pos="+pos
+        url = "http://"+host+":8888/read?filepath="+filepath+"&uid="+uid+"&gid="+gid+"&pos="+pos+"&size="+size
         print(url) 
         return url
 
@@ -27,25 +30,31 @@ def chunky(path, chunk):
    total_downloaded += len(chunk)
    print("chunk size",len(chunk))
    # the OS blocks on file reads + writes -- beware how big the chunks is as it could effect things
-   with open(path, 'a+b') as f:
+   with open(path, 'ab') as f:
        f.write(chunk)
 
 @gen.coroutine
-def writer(action,host,filepath,uid,gid,pos):
+def writer(action,host,filepath,targetdir,uid,gid,pos,size):
    print("writer function")
 #   tornado.ioloop.IOLoop.instance().start()
-   file_name = os.path.basename(filepath)+"_new"
-   f = open(os.path.basename(filepath)+"_new",'w')
+   file_name = targetdir+os.path.basename(filepath)
+   if os.path.exists(targetdir):
+       pass
+   else:
+       os.mkdir(targetdir)
+   f = open(file_name,'w')
    f.close()
-   request = HTTPRequest(geturl(action,host,filepath,uid,gid,pos), streaming_callback=partial(chunky, file_name))
-   http_client = AsyncHTTPClient()
+   request = HTTPRequest(geturl(action,host,filepath,uid,gid,pos,size), streaming_callback=partial(chunky, file_name))
+   AsyncHTTPClient.configure('tornado.simple_httpclient.SimpleAsyncHTTPClient', max_body_size=512*1024*1024)
+   http_client = AsyncHTTPClient(force_instance=True)
+   #http_client.configure("tornado.simple_httpclient.SimpleAsyncHTTPClient",max_body_size=524288000)
    response = yield http_client.fetch(request)
    tornado.ioloop.IOLoop.instance().stop()
    print("total bytes downloaded was", total_downloaded)
 
-def entrance(action,host,filepath,uid,gid,pos):
-   print("entrance function")
-   writer(action,host,filepath,uid,gid,pos)
+def entrance(action,host,filepath,targetdir,uid,gid,pos,size):
+   #print("entrance function")
+   writer(action,host,filepath,targetdir,uid,gid,pos,size)
    tornado.ioloop.IOLoop.instance().start()
 
 

main.cpp

@@ -1,7 +1,7 @@
 #include "client.h"
 int main()
 {
-   transfer("read","192.168.83.218","/dev/shm/500M.file","0","0","138");
+   transfer("read","192.168.83.218","/dev/shm/500M.file","/root/leaf/pytoc/download/","0","0","138","524200000");
    return 0;
 
 }

server.py

@@ -13,14 +13,23 @@ def read_in_chunks(infile, chunk_size=1024*1024):
        yield chunk
        chunk = infile.read(chunk_size)
 
-def read_in_chunks_pos(base_dir, pos, chunk_size=1024*1024):
+def read_in_chunks_pos(base_dir, pos, size, chunk_size=1024*1024):
    with open(base_dir, 'rb') as infile:
      infile.seek(int(pos))
+     no = int(size) // chunk_size
+     i = 0
+     #chunk = infile.read(chunk_size)
+     #while chunk and i<no:
+     while i<no:
         chunk = infile.read(chunk_size)
-     while chunk:
         yield chunk
-       chunk = infile.read(chunk_size)
+        i = i+1
+     if (int(size) % chunk_size) != 0:
+        last = int(size) % chunk_size
+        chunk = infile.read(last)
+        yield chunk
    infile.close()
+
 class ReadRequestHandler(tornado.web.RequestHandler):
    @gen.coroutine
    def get(self):
@@ -29,10 +38,10 @@ class ReadRequestHandler(tornado.web.RequestHandler):
         gid = self.get_argument('gid')
         base_dir = self.get_argument('filepath')
         pos = self.get_argument('pos')
-
+        size = self.get_argument('size')
         # Python protocol does not require () on it's if statements like you are
 
-        if base_dir==None or uid==None or gid==None or pos==None:
+        if base_dir==None or uid==None or gid==None or pos==None or size==None:
             self.write("Invalid argument!You caused a %d error."%status_code)
             exit(1)
         if os.path.exists(base_dir):
@@ -50,12 +59,11 @@ class ReadRequestHandler(tornado.web.RequestHandler):
             exit(1)
         else:
           #  with open(base_dir, 'rb') as infile:
-                for chunk in read_in_chunks_pos(base_dir,pos):
+                for chunk in read_in_chunks_pos(base_dir,pos,size):
                     self.write(chunk)
                     yield gen.Task(self.flush)
                     total_sent += len(chunk)
                     print("sent",total_sent)
-
                 self.finish()
 class ListRequestHandler(tornado.web.RequestHandler):
    @tornado.web.asynchronous
@@ -140,6 +148,6 @@ if __name__ == "__main__":
     (r"/list",ListRequestHandler),
     (r"/read",ReadRequestHandler)
     ])
-   application.listen(8880)
+   application.listen(8888)
    tornado.ioloop.IOLoop.instance().start()