From d4111cb49581935d169c5eccc95aab60c69f9bdf Mon Sep 17 00:00:00 2001
From: root <root@vm083218.v.ihep.ac.cn>
Date: Mon, 19 Dec 2016 14:11:17 +0800
Subject: [PATCH] with size and targetdir

---
 client.cpp   |  16 +++++++++-------
 client.h     |   2 +-
 client.py    |  31 ++++++++++++++++++++-----------
 libclient.so | Bin 13285 -> 13291 bytes
 main.cpp     |   2 +-
 server.py    |  30 +++++++++++++++++++-----------
 6 files changed, 50 insertions(+), 31 deletions(-)

diff --git a/client.cpp b/client.cpp
index ff19c8a..0072faa 100644
--- a/client.cpp
+++ b/client.cpp
@@ -1,9 +1,9 @@
 #include "client.h"
-void transfer(const char *action,const char *host,const char *filepath,const char *uid,const char *gid,const char *position)
+void transfer(const char *action,const char *host,const char *filepath,const char *targetdir,const char *uid,const char *gid,const char *position,const char *size)
 {
  
  Py_Initialize();
- if(Py_IsInitialized())
+ if(!Py_IsInitialized())
  {
    printf("can't initialize\n");
  }
@@ -14,17 +14,19 @@ void transfer(const char *action,const char *host,const char *filepath,const cha
  PyRun_SimpleString("import os");
  PyRun_SimpleString("import string");
  PyRun_SimpleString("sys.path.append('./')");
- PyObject *pArgs = PyTuple_New(6);   
+ PyObject *pArgs = PyTuple_New(8);   
  PyTuple_SetItem(pArgs,0,Py_BuildValue("s",action));  
  PyTuple_SetItem(pArgs,1,Py_BuildValue("s",host));  
  PyTuple_SetItem(pArgs,2,Py_BuildValue("s",filepath));
- PyTuple_SetItem(pArgs,3,Py_BuildValue("s",uid));  
- PyTuple_SetItem(pArgs,4,Py_BuildValue("s",gid));  
- PyTuple_SetItem(pArgs,5,Py_BuildValue("s",position));  
+ PyTuple_SetItem(pArgs,3,Py_BuildValue("s",targetdir));
+ PyTuple_SetItem(pArgs,4,Py_BuildValue("s",uid));  
+ PyTuple_SetItem(pArgs,5,Py_BuildValue("s",gid));  
+ PyTuple_SetItem(pArgs,6,Py_BuildValue("s",position));  
+ PyTuple_SetItem(pArgs,7,Py_BuildValue("s",size));  
  pModule = PyImport_ImportModule("client");
  if(pModule == NULL)
  {
-   printf("can't python file\n");
+   printf("can't load module\n");
  }
  pFunc = PyObject_GetAttrString(pModule,"entrance");
  if(pFunc == NULL)
diff --git a/client.h b/client.h
index 0750a2f..acfea88 100644
--- a/client.h
+++ b/client.h
@@ -3,5 +3,5 @@
 #include <iostream>
 #include <Python.h>
 using namespace std;
-void transfer(const char *action,const char *host,const char *filepath,const char *uid,const char *gid,const char *position);
+void transfer(const char *action,const char *host,const char *filepath,const char *targetdir,const char *uid,const char *gid,const char *position,const char *size);
 #endif
diff --git a/client.py b/client.py
index 51a2568..7f50bea 100644
--- a/client.py
+++ b/client.py
@@ -1,9 +1,12 @@
 from tornado.httpclient import HTTPRequest, AsyncHTTPClient
+from tornado.simple_httpclient import SimpleAsyncHTTPClient
 import tornado.ioloop
 import tornado.web
 import os,sys
 from tornado import gen
 from functools import partial
+#import time,datatime
+
 
 total_downloaded = 0
 #action = sys.argv[1]
@@ -11,12 +14,12 @@ total_downloaded = 0
 #uid = sys.argv[3]
 #gid = sys.argv[4]
 #pos = sys.argv[5]
-def geturl(action,host,filepath,uid,gid,pos):
+def geturl(action,host,filepath,uid,gid,pos,size):
     if action=="read":
  #       uid = sys.argv[3]
   #      gid = sys.argv[4]
    #     pos = sys.argv[5]
-        url = "http://"+host+":8880/read?filepath="+filepath+"&uid="+uid+"&gid="+gid+"&pos="+pos
+        url = "http://"+host+":8888/read?filepath="+filepath+"&uid="+uid+"&gid="+gid+"&pos="+pos+"&size="+size
         print(url) 
         return url
 
@@ -27,25 +30,31 @@ def chunky(path, chunk):
    total_downloaded += len(chunk)
    print("chunk size",len(chunk))
    # the OS blocks on file reads + writes -- beware how big the chunks is as it could effect things
-   with open(path, 'a+b') as f:
+   with open(path, 'ab') as f:
        f.write(chunk)
 
 @gen.coroutine
-def writer(action,host,filepath,uid,gid,pos):
+def writer(action,host,filepath,targetdir,uid,gid,pos,size):
    print("writer function")
 #   tornado.ioloop.IOLoop.instance().start()
-   file_name = os.path.basename(filepath)+"_new"
-   f = open(os.path.basename(filepath)+"_new",'w')
+   file_name = targetdir+os.path.basename(filepath)
+   if os.path.exists(targetdir):
+       pass
+   else:
+       os.mkdir(targetdir)
+   f = open(file_name,'w')
    f.close()
-   request = HTTPRequest(geturl(action,host,filepath,uid,gid,pos), streaming_callback=partial(chunky, file_name))
-   http_client = AsyncHTTPClient()
+   request = HTTPRequest(geturl(action,host,filepath,uid,gid,pos,size), streaming_callback=partial(chunky, file_name))
+   AsyncHTTPClient.configure('tornado.simple_httpclient.SimpleAsyncHTTPClient', max_body_size=512*1024*1024)
+   http_client = AsyncHTTPClient(force_instance=True)
+   #http_client.configure("tornado.simple_httpclient.SimpleAsyncHTTPClient",max_body_size=524288000)
    response = yield http_client.fetch(request)
    tornado.ioloop.IOLoop.instance().stop()
    print("total bytes downloaded was", total_downloaded)
 
-def entrance(action,host,filepath,uid,gid,pos):
-   print("entrance function")
-   writer(action,host,filepath,uid,gid,pos)
+def entrance(action,host,filepath,targetdir,uid,gid,pos,size):
+   #print("entrance function")
+   writer(action,host,filepath,targetdir,uid,gid,pos,size)
    tornado.ioloop.IOLoop.instance().start()
 
 
diff --git a/libclient.so b/libclient.so
index 5bcd6d73b65786bfa0794b6f60c74fa39dc32803..a59be23fd56fa0bcefe42f9622d04b82c402e1e7 100755
GIT binary patch
delta 2262
zcmZuz4NOy46u$SBVhjC6u&n}p9aO0xB`%0RYh*#<+ZIFpVd^5$5~GqZe-?<FnJW%h
z!jK4@Wz&Uh=tedcmoUjTHA6wC60<E%;^vkusZO(2O{l0zMO=5zZ95s;fp_lvzWd#C
z&;5D#_U6{xTidO?sKd5gJ7H$RCOT?PrxnD(lM)v557JL~f4esvSfMm)8Y0zgM-5Gf
zN6x_=Zp~{*Ds|bg`7C_0@j3ZN=j&zVSMFThwJvju@xa9sZFdc~M@9|c<FZ=Cv@8fK
zrKS8hbVwFn07K|+2%io^^DKM<?n-XH1{|v8>gPFQW^6Z|9}?<o!e<EeRpAE&6D1Gr
z^Q`=ez@x(N7m82l-L^+!%)~70Q(49opV(G7oMd@vd%U)LN^Ly~(^x9A>SHaOs#s|f
zJdPwKh7*2<;B~Bv1i~OqeX##y^)hJ*V;uh$h@Kdev9`|DsbNgT{K%0}eO0`kaQ)00
zEn^aE1&79B3+RYrtUF5<B{mh-3@bGj-rZlLc|-DlpDr_{d9Y3$%)v^f@mtN=xGY{o
zi^otBb>2sBw3p%=;cZkK3PjPIw+Q3Bv(P1huM?&WLxTeUN|@#$)F<#|!bya?qkd$Z
z7l}|4(JlmM2qzP65cnkFX@qwQe2g%SCbUW5&j{ndS;#B!5yCXlp;Ce0C!9*SNZ?Mw
zG&v#JFNn8^pcM<51b&mSfiM$z4`CzW@LxzcUm;8@778F%+SEIdP})4zSX3&mh|+RX
zX^a?lOI2r;R&~?#*c6XdMFTX0s9zM*&rgR9I)}eP2rk!+RVgj!qt{8(;CYB7tDmDZ
zdP?Z65@Xisu#a(*%z^{hZEzc=QgvQ^39I?Nvs=gtjrJx?ix$!|j6zm78x#0BEtbH{
zN^E}&X^a)UR25o_0gUzrn9waseP^&~P{d!Yf>QmQ7nk6I1-;nKDG{#CQ%uCtNkkzL
zg_DSUBC?P;O=}Y@!#Aa+JfO6e597G7Ygq*Y`nmi*1oioR7}5>-ybDUv?}S=IzO+n&
zLk5Sxwcdo<S-63YQ`f;W6BOqRh6$XaO`)`02xdP<uC?46i#v*&FQ$EWl9o0MX~&3b
z^L&S*;8&u)Jfbx2jj(!}&TohweI&v~l(yni3z48^8)DKhO2aYf=OjHRq$_At;ggh~
z(((hz2OdGD(VMf1TDPjx#Ux({?icC|y?{TFZu}7(FqZLV7%@6?%a9M=8mIg+UeL?9
zGm48y+N!QJVuAAC$05^XleV<OLQ}Q2c871X{PK>?_3)9&<;=t0=`*_VFdAKk<sIy-
z7E3cO>wYZPV|f%Fm~!|*u$kvLUd4KG9zAO?y(N|DlU1@xH%-GbQM(zRz2Gz3c~Ad7
z^Vg~7#Np8#KbX3qH_PsG{B!W29*qtsG90zA@+r7t3Z}z~j6OcH#BmruEm;@1vi<x$
z=*_M$d+Cctqje-J0<GQ6!?4J1=Y6o=ei~%CfzR$gBv(k-B=nS9+ZWgNfg`6x8;+we
z_z)G*rW9cgBT5AK!Zk#RG=kvBEzz3js6aD;FSo>86wftL)<SQtTbnqqyWkqO=!s8K
z4>+79DU+_jYUhHKb>>*Ob>{vf&SVvz1-*G6;s|`M3{^72<Xje8BDt6{=2c9?ywHKQ
ziS+(mn}P`rQQO31PO4Vn40pj@mxcGB&&V{eL{gJPc##+xX0>>3HvdnQLKI4Pk!^xq
z`FL4COa5{mfFJWW+C1^WD)CSfE<P!n;kg1gkHBjMqxf0Z%o~9<h3WhkaYg?r`~%^2
z@I|48TVWsiT;ZP)eifMA7QU9=A<u2K?Ze~b|E|&=q5hJ5)??I|*bFxH8cxFRB8`3q
hXfx=IenRvQxkn^8Hh%y*=Vv9aF4~Cyu>I-Xe*<FTcfkMv

delta 2188
zcmZWqZ%k8H6u<YCVjrb{6-!0zt5Bh!)MykmVTee0ewfx6CR;)jvuXLVp()!~+`yqG
zl{5j*Ow3{w{Um(Y(0(u@aeyvLwz+i}H^Y*xxENb!BIq_d|8?iwzD~uHymR00oZmV3
z+;iW%_s`w8cK7CUrO(+QAGa|f$NO4G?Bmuncs{~feq!F*$#?$NyAC%E{Jv)B;sd*!
z`smzKxXbfP4OxEKi6I5}6yh`d$B}(?_1Ev-*weJ|tMH8Z&b52yq6=ynv~WeQV={}u
z7O94xgFeZ@m%te6--I^9*fIzI2WBK6-vn-5L)He)7`V&kN(==1SD`I}Jt6dYVfk0+
zF=6>pyeEV{BrK=s-5Hb^voZ%eqbOM9b7_H7Sq^tV^Y+j3QZMYH=9w(V;wr5*ndo-m
zH8O)KZHY=?*B^@v>0gt!aK_nkk(Dt8-9hgW17kYYfi@EIZ`1sQ+ZVLTj7jXE=08@b
zkeOkSkSNp|98N{6;LI#<Z)A_*6RGq1JcX$TM}zt}?S|U@jp0%{I<-p5Q&`HAh1%j;
zhi`-f<V?g;D4s_c8o++DL{#AWgsGOqsK9p!>j)1Ce3Nh%;eLT95qAdkWat%!2_j?@
z?h^Q0!gC0>34D<-tuC=c;9<hp0+#R#e2#Dq;dKHJ5;hWEF7Ro>xr7ygKcA;`vH<Nw
z!YT|$h+rnn1U^jILU{Ta8a$s8rhX=3h}E#Z3k_<xdM^%*syC^IZmHc#^9NG%CACNY
z+1zw%?>DDn)IsELO!L?O&xc^Hvva#3P6Y2atD(!Oo5by^mXX8SOVsY_26SWk(5+xD
z&Xr?RSo{J`9}i$j&6o8tbn^$7c9B$ATnDf`+Q{k~RtyU+!9s4^(}gT(#Q2M7#$2gr
zb7C7Nu(-M)*R;lXbo9We$e)tHZz_4?Hg06xkJEfD(uJ$1$WZfwVUi3LFBpCyLm?VQ
zXb<Cc_@;*HV`@+RIIat){$7HsrZV0SNmDrwLb17=tKdhy5cZkNrD+ZWX7|o;^;P_+
z;>M>UJgIhnoMi2^@(X0%@C400^+`3nY6P(wx)MK2rcnJfGN;qbFNnE7Fkh$nr-#KN
zLO&4Q{si`0{6$+Sx<}t3+Hoa5^_cSJ?G^B6;%$Bce_87IBve^F#b?kSuOUu2Z^(ol
z46iy$)E@n?1qY<BcntfkPMyWY;J9_C+}05cy!WoMs}<(hyxyfaKYjUCk1zv{>#(*q
z9O<*`#jy#;AsiVr+KPA_I&3BGahxrREoHT4>mjMml&$+t%AI3i3(@p8J|DuE&Bgm8
zQ?~O)u-bRnqV#E_Wd8VG7$|hL6i^BjT&@s0VMsE#W)|K!3)6{1q9tcB`oU1N1B_LH
zcTp$rgMmexZ7ThYDD=;gl!EN?;d0ivTznATbd5;!jbK*}Mfw$0!kENjpFF6M2f<xb
zDaSPwhZaPo+EG>trx0bbt1yKqQ%4Nk#g#In-<$+miYskOrW4wj4+F(MIdc|Q!4yW+
zTGIiw;PzDJyl{2x^sLTFYR;rJa@v!v<9RSpdJ>n=;?2`#Gc3;QaAt~2r%7ua9=<B{
zp*K_B%U<jqjG;GE;LF}zZ03yD!D~_H>C9|gj24!pF|w&Fhrbe7ff*!RydU<IJ9saI
z${TnTek$MYyg}y{3Vm)Q5^gv#71+4c$B#kB(tErLT9;kRx1;s>eUxU$aM#oK=tFVG
z>fvmKgU==HEW<OM?#0<R-)g~GFZ5fajcpneZ%`=kc{m8)Nuj?a?R+R0^IysJjJ&c1
Z{~2+YM#g;CB-pyL0Qy#pLi@`1{{t}aRq_A;

diff --git a/main.cpp b/main.cpp
index 5b50526..5deba35 100644
--- a/main.cpp
+++ b/main.cpp
@@ -1,7 +1,7 @@
 #include "client.h"
 int main()
 {
-   transfer("read","192.168.83.218","/dev/shm/500M.file","0","0","138");
+   transfer("read","192.168.83.218","/dev/shm/500M.file","/root/leaf/pytoc/download/","0","0","138","524200000");
    return 0;
 
 }
diff --git a/server.py b/server.py
index 4f40327..595d89d 100644
--- a/server.py
+++ b/server.py
@@ -13,14 +13,23 @@ def read_in_chunks(infile, chunk_size=1024*1024):
        yield chunk
        chunk = infile.read(chunk_size)
 
-def read_in_chunks_pos(base_dir, pos, chunk_size=1024*1024):
+def read_in_chunks_pos(base_dir, pos, size, chunk_size=1024*1024):
    with open(base_dir, 'rb') as infile:
      infile.seek(int(pos))
-     chunk = infile.read(chunk_size)
-     while chunk:
-       yield chunk
-       chunk = infile.read(chunk_size)
-     infile.close()
+     no = int(size) // chunk_size
+     i = 0
+     #chunk = infile.read(chunk_size)
+     #while chunk and i<no:
+     while i<no:
+        chunk = infile.read(chunk_size)
+        yield chunk
+        i = i+1
+     if (int(size) % chunk_size) != 0:
+        last = int(size) % chunk_size
+        chunk = infile.read(last)
+        yield chunk
+   infile.close()
+
 class ReadRequestHandler(tornado.web.RequestHandler):
    @gen.coroutine
    def get(self):
@@ -29,10 +38,10 @@ class ReadRequestHandler(tornado.web.RequestHandler):
         gid = self.get_argument('gid')
         base_dir = self.get_argument('filepath')
         pos = self.get_argument('pos')
-
+        size = self.get_argument('size')
         # Python protocol does not require () on it's if statements like you are
 
-        if base_dir==None or uid==None or gid==None or pos==None:
+        if base_dir==None or uid==None or gid==None or pos==None or size==None:
             self.write("Invalid argument!You caused a %d error."%status_code)
             exit(1)
         if os.path.exists(base_dir):
@@ -50,12 +59,11 @@ class ReadRequestHandler(tornado.web.RequestHandler):
             exit(1)
         else:
           #  with open(base_dir, 'rb') as infile:
-                for chunk in read_in_chunks_pos(base_dir,pos):
+                for chunk in read_in_chunks_pos(base_dir,pos,size):
                     self.write(chunk)
                     yield gen.Task(self.flush)
                     total_sent += len(chunk)
                     print("sent",total_sent)
-
                 self.finish()
 class ListRequestHandler(tornado.web.RequestHandler):
    @tornado.web.asynchronous
@@ -140,6 +148,6 @@ if __name__ == "__main__":
     (r"/list",ListRequestHandler),
     (r"/read",ReadRequestHandler)
     ])
-   application.listen(8880)
+   application.listen(8888)
    tornado.ioloop.IOLoop.instance().start()