problem with chunk size
This commit is contained in:
root
commit
51cb782784
|
|
@ -0,0 +1,42 @@
|
||||||
|
#include "client.h"
|
||||||
|
void transfer(const char *action,const char *host,const char *filepath,const char *uid,const char *gid,const char *position)
|
||||||
|
{
|
||||||
|
|
||||||
|
Py_Initialize();
|
||||||
|
if(Py_IsInitialized())
|
||||||
|
{
|
||||||
|
printf("can't initialize\n");
|
||||||
|
}
|
||||||
|
PyObject * pModule = NULL;
|
||||||
|
PyObject * pFunc = NULL;
|
||||||
|
PyObject * result = NULL;
|
||||||
|
PyRun_SimpleString("import sys");
|
||||||
|
PyRun_SimpleString("import os");
|
||||||
|
PyRun_SimpleString("import string");
|
||||||
|
PyRun_SimpleString("sys.path.append('./')");
|
||||||
|
PyObject *pArgs = PyTuple_New(6);
|
||||||
|
PyTuple_SetItem(pArgs,0,Py_BuildValue("s",action));
|
||||||
|
PyTuple_SetItem(pArgs,1,Py_BuildValue("s",host));
|
||||||
|
PyTuple_SetItem(pArgs,2,Py_BuildValue("s",filepath));
|
||||||
|
PyTuple_SetItem(pArgs,3,Py_BuildValue("s",uid));
|
||||||
|
PyTuple_SetItem(pArgs,4,Py_BuildValue("s",gid));
|
||||||
|
PyTuple_SetItem(pArgs,5,Py_BuildValue("s",position));
|
||||||
|
pModule = PyImport_ImportModule("client");
|
||||||
|
if(pModule == NULL)
|
||||||
|
{
|
||||||
|
printf("can't python file\n");
|
||||||
|
}
|
||||||
|
pFunc = PyObject_GetAttrString(pModule,"entrance");
|
||||||
|
if(pFunc == NULL)
|
||||||
|
{
|
||||||
|
printf("can't load function\n");
|
||||||
|
}
|
||||||
|
result = PyEval_CallObject(pFunc,pArgs);
|
||||||
|
Py_DECREF(pArgs);
|
||||||
|
if(result == NULL)
|
||||||
|
cout<<"null"<<endl;
|
||||||
|
Py_DECREF(result);
|
||||||
|
Py_Finalize();
|
||||||
|
//return 0;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
#ifndef _CLIENT_HEADER_
|
||||||
|
#define _CLIENT_HEADER_
|
||||||
|
#include <iostream>
|
||||||
|
#include <Python.h>
|
||||||
|
using namespace std;
|
||||||
|
void transfer(const char *action,const char *host,const char *filepath,const char *uid,const char *gid,const char *position);
|
||||||
|
#endif
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
from tornado.httpclient import HTTPRequest, AsyncHTTPClient
|
||||||
|
import tornado.ioloop
|
||||||
|
import tornado.web
|
||||||
|
import os,sys
|
||||||
|
from tornado import gen
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
|
total_downloaded = 0
|
||||||
|
#action = sys.argv[1]
|
||||||
|
#filepath = sys.argv[2]
|
||||||
|
#uid = sys.argv[3]
|
||||||
|
#gid = sys.argv[4]
|
||||||
|
#pos = sys.argv[5]
|
||||||
|
def geturl(action,host,filepath,uid,gid,pos):
|
||||||
|
if action=="read":
|
||||||
|
# uid = sys.argv[3]
|
||||||
|
# gid = sys.argv[4]
|
||||||
|
# pos = sys.argv[5]
|
||||||
|
url = "http://"+host+":8880/read?filepath="+filepath+"&uid="+uid+"&gid="+gid+"&pos="+pos
|
||||||
|
print(url)
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
def chunky(path, chunk):
|
||||||
|
# print("self._max_body_size",self._max_body_size)
|
||||||
|
global total_downloaded
|
||||||
|
total_downloaded += len(chunk)
|
||||||
|
print("chunk size",len(chunk))
|
||||||
|
# the OS blocks on file reads + writes -- beware how big the chunks is as it could effect things
|
||||||
|
with open(path, 'a+b') as f:
|
||||||
|
f.write(chunk)
|
||||||
|
|
||||||
|
@gen.coroutine
|
||||||
|
def writer(action,host,filepath,uid,gid,pos):
|
||||||
|
print("writer function")
|
||||||
|
# tornado.ioloop.IOLoop.instance().start()
|
||||||
|
file_name = os.path.basename(filepath)+"_new"
|
||||||
|
f = open(os.path.basename(filepath)+"_new",'w')
|
||||||
|
f.close()
|
||||||
|
request = HTTPRequest(geturl(action,host,filepath,uid,gid,pos), streaming_callback=partial(chunky, file_name))
|
||||||
|
http_client = AsyncHTTPClient()
|
||||||
|
response = yield http_client.fetch(request)
|
||||||
|
tornado.ioloop.IOLoop.instance().stop()
|
||||||
|
print("total bytes downloaded was", total_downloaded)
|
||||||
|
|
||||||
|
def entrance(action,host,filepath,uid,gid,pos):
|
||||||
|
print("entrance function")
|
||||||
|
writer(action,host,filepath,uid,gid,pos)
|
||||||
|
tornado.ioloop.IOLoop.instance().start()
|
||||||
|
|
||||||
|
|
||||||
Binary file not shown.
|
|
@ -0,0 +1,7 @@
|
||||||
|
#include "client.h"
|
||||||
|
int main()
|
||||||
|
{
|
||||||
|
transfer("read","192.168.83.218","/dev/shm/500M.file","0","0","138");
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,145 @@
|
||||||
|
import tornado.ioloop
|
||||||
|
import tornado.web
|
||||||
|
import tornado.options
|
||||||
|
from tornado import gen
|
||||||
|
import os,sys
|
||||||
|
from stat import *
|
||||||
|
|
||||||
|
GB = 1024 * 1024 * 1024
|
||||||
|
|
||||||
|
def read_in_chunks(infile, chunk_size=1024*1024):
|
||||||
|
chunk = infile.read(chunk_size)
|
||||||
|
while chunk:
|
||||||
|
yield chunk
|
||||||
|
chunk = infile.read(chunk_size)
|
||||||
|
|
||||||
|
def read_in_chunks_pos(base_dir, pos, chunk_size=1024*1024):
|
||||||
|
with open(base_dir, 'rb') as infile:
|
||||||
|
infile.seek(int(pos))
|
||||||
|
chunk = infile.read(chunk_size)
|
||||||
|
while chunk:
|
||||||
|
yield chunk
|
||||||
|
chunk = infile.read(chunk_size)
|
||||||
|
infile.close()
|
||||||
|
class ReadRequestHandler(tornado.web.RequestHandler):
|
||||||
|
@gen.coroutine
|
||||||
|
def get(self):
|
||||||
|
total_sent = 0
|
||||||
|
uid = self.get_argument('uid')
|
||||||
|
gid = self.get_argument('gid')
|
||||||
|
base_dir = self.get_argument('filepath')
|
||||||
|
pos = self.get_argument('pos')
|
||||||
|
|
||||||
|
# Python protocol does not require () on it's if statements like you are
|
||||||
|
|
||||||
|
if base_dir==None or uid==None or gid==None or pos==None:
|
||||||
|
self.write("Invalid argument!You caused a %d error."%status_code)
|
||||||
|
exit(1)
|
||||||
|
if os.path.exists(base_dir):
|
||||||
|
statinfo = os.stat(base_dir)
|
||||||
|
if(int(uid)==statinfo.st_uid and int(gid)==statinfo.st_gid):
|
||||||
|
mode = statinfo.st_mode
|
||||||
|
else:
|
||||||
|
self.write("Permission denied.")
|
||||||
|
exit(1)
|
||||||
|
else:
|
||||||
|
self.write("File or directory doesn't exist!You caused a %d error."%status_code)
|
||||||
|
exit(1)
|
||||||
|
if (S_ISDIR(mode)):
|
||||||
|
self.write("This is not a file!You caused a %d error."%status_code)
|
||||||
|
exit(1)
|
||||||
|
else:
|
||||||
|
# with open(base_dir, 'rb') as infile:
|
||||||
|
for chunk in read_in_chunks_pos(base_dir,pos):
|
||||||
|
self.write(chunk)
|
||||||
|
yield gen.Task(self.flush)
|
||||||
|
total_sent += len(chunk)
|
||||||
|
print("sent",total_sent)
|
||||||
|
|
||||||
|
self.finish()
|
||||||
|
class ListRequestHandler(tornado.web.RequestHandler):
|
||||||
|
@tornado.web.asynchronous
|
||||||
|
@gen.coroutine
|
||||||
|
def get(self):
|
||||||
|
uid = self.get_argument('uid')
|
||||||
|
gid = self.get_argument('gid')
|
||||||
|
base_dir = self.get_argument('path')
|
||||||
|
if (base_dir==None or uid==None or gid==None):
|
||||||
|
self.write("Invalid argument!You caused a %d error."%status_code)
|
||||||
|
exit(1)
|
||||||
|
if(os.path.exists(base_dir)):
|
||||||
|
statinfo = os.stat(base_dir)
|
||||||
|
self.write('{'+'"father_node"'+':')
|
||||||
|
statdict = {'path':base_dir,'mode':str(statinfo.st_mode),'ino':str(statinfo.st_ino),'dev':str(statinfo.st_dev),'nlink':str(statinfo.st_nlink),'uid':str(statinfo.st_uid),'gid':str(statinfo.st_gid),'size':str(statinfo.st_size),'atime':str(statinfo.st_atime),'mtime':str(statinfo.st_mtime),'ctime':str(statinfo.st_ctime)}
|
||||||
|
if(int(uid)==statinfo.st_uid and int(gid)==statinfo.st_gid):
|
||||||
|
self.write(statdict)
|
||||||
|
mode = statinfo.st_mode
|
||||||
|
else:
|
||||||
|
self.write("Permission denied.")
|
||||||
|
exit(1)
|
||||||
|
else:
|
||||||
|
self.write("File or directory doesn't exist!You caused a %d error."%status_code)
|
||||||
|
exit(1)
|
||||||
|
if (S_ISDIR(mode)==None):
|
||||||
|
self.write("This is not a directory!You caused a %d error."%status_code)
|
||||||
|
exit(1)
|
||||||
|
else:
|
||||||
|
files = os.listdir(base_dir)
|
||||||
|
for f in files:
|
||||||
|
statinfo = os.stat(base_dir + '/' +f)
|
||||||
|
self.write(',"'+f+'":')
|
||||||
|
statdict = {'path':(base_dir + '/' +f),'mode':str(statinfo.st_mode),'ino':str(statinfo.st_ino),'dev':str(statinfo.st_dev),'nlink':str(statinfo.st_nlink),'uid':str(statinfo.st_uid),'gid':str(statinfo.st_gid),'size':str(statinfo.st_size),'atime':str(statinfo.st_atime),'mtime':str(statinfo.st_mtime),'ctime':str(statinfo.st_ctime)}
|
||||||
|
# print ("statdict",statdict)
|
||||||
|
self.write(statdict)
|
||||||
|
self.write("}")
|
||||||
|
def write_error(self,status_code,**kwargs):
|
||||||
|
self.write("Gosh darnit,user!You caused a %d error."%status_code)
|
||||||
|
|
||||||
|
class StreamingRequestHandler(tornado.web.RequestHandler):
|
||||||
|
@tornado.web.asynchronous
|
||||||
|
@gen.coroutine
|
||||||
|
def get(self):
|
||||||
|
# back as right now this is limited to what is hard coded in
|
||||||
|
total_sent = 0
|
||||||
|
uid = self.get_argument('uid')
|
||||||
|
gid = self.get_argument('gid')
|
||||||
|
base_dir = self.get_argument('filepath')
|
||||||
|
if (base_dir==None or uid==None or gid==None):
|
||||||
|
self.write("Invalid argument!You caused a %d error."%status_code)
|
||||||
|
exit(1)
|
||||||
|
if(os.path.exists(base_dir)):
|
||||||
|
statinfo = os.stat(base_dir)
|
||||||
|
if(int(uid)==statinfo.st_uid and int(gid)==statinfo.st_gid):
|
||||||
|
mode = statinfo.st_mode
|
||||||
|
else:
|
||||||
|
self.write("Permission denied.")
|
||||||
|
exit(1)
|
||||||
|
else:
|
||||||
|
self.write("File or directory doesn't exist!You caused a %d error."%status_code)
|
||||||
|
exit(1)
|
||||||
|
if (S_ISDIR(mode)):
|
||||||
|
self.write("This is not a file!You caused a %d error."%status_code)
|
||||||
|
exit(1)
|
||||||
|
else:
|
||||||
|
with open(base_dir, 'rb') as infile:
|
||||||
|
for chunk in read_in_chunks(infile):
|
||||||
|
self.write(chunk)
|
||||||
|
yield gen.Task(self.flush)
|
||||||
|
total_sent += len(chunk)
|
||||||
|
print("sent",total_sent)
|
||||||
|
|
||||||
|
self.finish()
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# this was connected to the pyCurl call and as far as I know now not
|
||||||
|
# beng used so try without to insure it's no longer needed
|
||||||
|
# tornado.options.parse_command_line()
|
||||||
|
print (tornado.version)
|
||||||
|
application = tornado.web.Application([
|
||||||
|
(r"/download", StreamingRequestHandler),
|
||||||
|
(r"/list",ListRequestHandler),
|
||||||
|
(r"/read",ReadRequestHandler)
|
||||||
|
])
|
||||||
|
application.listen(8880)
|
||||||
|
tornado.ioloop.IOLoop.instance().start()
|
||||||
|
|
||||||
Loading…
Reference in New Issue