problem with chunk size
This commit is contained in:
root
commit
51cb782784
|
|
@ -0,0 +1,42 @@
|
|||
#include "client.h"
|
||||
void transfer(const char *action,const char *host,const char *filepath,const char *uid,const char *gid,const char *position)
|
||||
{
|
||||
|
||||
Py_Initialize();
|
||||
if(Py_IsInitialized())
|
||||
{
|
||||
printf("can't initialize\n");
|
||||
}
|
||||
PyObject * pModule = NULL;
|
||||
PyObject * pFunc = NULL;
|
||||
PyObject * result = NULL;
|
||||
PyRun_SimpleString("import sys");
|
||||
PyRun_SimpleString("import os");
|
||||
PyRun_SimpleString("import string");
|
||||
PyRun_SimpleString("sys.path.append('./')");
|
||||
PyObject *pArgs = PyTuple_New(6);
|
||||
PyTuple_SetItem(pArgs,0,Py_BuildValue("s",action));
|
||||
PyTuple_SetItem(pArgs,1,Py_BuildValue("s",host));
|
||||
PyTuple_SetItem(pArgs,2,Py_BuildValue("s",filepath));
|
||||
PyTuple_SetItem(pArgs,3,Py_BuildValue("s",uid));
|
||||
PyTuple_SetItem(pArgs,4,Py_BuildValue("s",gid));
|
||||
PyTuple_SetItem(pArgs,5,Py_BuildValue("s",position));
|
||||
pModule = PyImport_ImportModule("client");
|
||||
if(pModule == NULL)
|
||||
{
|
||||
printf("can't python file\n");
|
||||
}
|
||||
pFunc = PyObject_GetAttrString(pModule,"entrance");
|
||||
if(pFunc == NULL)
|
||||
{
|
||||
printf("can't load function\n");
|
||||
}
|
||||
result = PyEval_CallObject(pFunc,pArgs);
|
||||
Py_DECREF(pArgs);
|
||||
if(result == NULL)
|
||||
cout<<"null"<<endl;
|
||||
Py_DECREF(result);
|
||||
Py_Finalize();
|
||||
//return 0;
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#ifndef _CLIENT_HEADER_
|
||||
#define _CLIENT_HEADER_
|
||||
#include <iostream>
|
||||
#include <Python.h>
|
||||
using namespace std;
|
||||
void transfer(const char *action,const char *host,const char *filepath,const char *uid,const char *gid,const char *position);
|
||||
#endif
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
from tornado.httpclient import HTTPRequest, AsyncHTTPClient
|
||||
import tornado.ioloop
|
||||
import tornado.web
|
||||
import os,sys
|
||||
from tornado import gen
|
||||
from functools import partial
|
||||
|
||||
total_downloaded = 0
|
||||
#action = sys.argv[1]
|
||||
#filepath = sys.argv[2]
|
||||
#uid = sys.argv[3]
|
||||
#gid = sys.argv[4]
|
||||
#pos = sys.argv[5]
|
||||
def geturl(action,host,filepath,uid,gid,pos):
|
||||
if action=="read":
|
||||
# uid = sys.argv[3]
|
||||
# gid = sys.argv[4]
|
||||
# pos = sys.argv[5]
|
||||
url = "http://"+host+":8880/read?filepath="+filepath+"&uid="+uid+"&gid="+gid+"&pos="+pos
|
||||
print(url)
|
||||
return url
|
||||
|
||||
|
||||
def chunky(path, chunk):
|
||||
# print("self._max_body_size",self._max_body_size)
|
||||
global total_downloaded
|
||||
total_downloaded += len(chunk)
|
||||
print("chunk size",len(chunk))
|
||||
# the OS blocks on file reads + writes -- beware how big the chunks is as it could effect things
|
||||
with open(path, 'a+b') as f:
|
||||
f.write(chunk)
|
||||
|
||||
@gen.coroutine
|
||||
def writer(action,host,filepath,uid,gid,pos):
|
||||
print("writer function")
|
||||
# tornado.ioloop.IOLoop.instance().start()
|
||||
file_name = os.path.basename(filepath)+"_new"
|
||||
f = open(os.path.basename(filepath)+"_new",'w')
|
||||
f.close()
|
||||
request = HTTPRequest(geturl(action,host,filepath,uid,gid,pos), streaming_callback=partial(chunky, file_name))
|
||||
http_client = AsyncHTTPClient()
|
||||
response = yield http_client.fetch(request)
|
||||
tornado.ioloop.IOLoop.instance().stop()
|
||||
print("total bytes downloaded was", total_downloaded)
|
||||
|
||||
def entrance(action,host,filepath,uid,gid,pos):
|
||||
print("entrance function")
|
||||
writer(action,host,filepath,uid,gid,pos)
|
||||
tornado.ioloop.IOLoop.instance().start()
|
||||
|
||||
|
||||
Binary file not shown.
|
|
@ -0,0 +1,7 @@
|
|||
#include "client.h"
|
||||
int main()
|
||||
{
|
||||
transfer("read","192.168.83.218","/dev/shm/500M.file","0","0","138");
|
||||
return 0;
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
import tornado.ioloop
|
||||
import tornado.web
|
||||
import tornado.options
|
||||
from tornado import gen
|
||||
import os,sys
|
||||
from stat import *
|
||||
|
||||
GB = 1024 * 1024 * 1024
|
||||
|
||||
def read_in_chunks(infile, chunk_size=1024*1024):
|
||||
chunk = infile.read(chunk_size)
|
||||
while chunk:
|
||||
yield chunk
|
||||
chunk = infile.read(chunk_size)
|
||||
|
||||
def read_in_chunks_pos(base_dir, pos, chunk_size=1024*1024):
|
||||
with open(base_dir, 'rb') as infile:
|
||||
infile.seek(int(pos))
|
||||
chunk = infile.read(chunk_size)
|
||||
while chunk:
|
||||
yield chunk
|
||||
chunk = infile.read(chunk_size)
|
||||
infile.close()
|
||||
class ReadRequestHandler(tornado.web.RequestHandler):
|
||||
@gen.coroutine
|
||||
def get(self):
|
||||
total_sent = 0
|
||||
uid = self.get_argument('uid')
|
||||
gid = self.get_argument('gid')
|
||||
base_dir = self.get_argument('filepath')
|
||||
pos = self.get_argument('pos')
|
||||
|
||||
# Python protocol does not require () on it's if statements like you are
|
||||
|
||||
if base_dir==None or uid==None or gid==None or pos==None:
|
||||
self.write("Invalid argument!You caused a %d error."%status_code)
|
||||
exit(1)
|
||||
if os.path.exists(base_dir):
|
||||
statinfo = os.stat(base_dir)
|
||||
if(int(uid)==statinfo.st_uid and int(gid)==statinfo.st_gid):
|
||||
mode = statinfo.st_mode
|
||||
else:
|
||||
self.write("Permission denied.")
|
||||
exit(1)
|
||||
else:
|
||||
self.write("File or directory doesn't exist!You caused a %d error."%status_code)
|
||||
exit(1)
|
||||
if (S_ISDIR(mode)):
|
||||
self.write("This is not a file!You caused a %d error."%status_code)
|
||||
exit(1)
|
||||
else:
|
||||
# with open(base_dir, 'rb') as infile:
|
||||
for chunk in read_in_chunks_pos(base_dir,pos):
|
||||
self.write(chunk)
|
||||
yield gen.Task(self.flush)
|
||||
total_sent += len(chunk)
|
||||
print("sent",total_sent)
|
||||
|
||||
self.finish()
|
||||
class ListRequestHandler(tornado.web.RequestHandler):
|
||||
@tornado.web.asynchronous
|
||||
@gen.coroutine
|
||||
def get(self):
|
||||
uid = self.get_argument('uid')
|
||||
gid = self.get_argument('gid')
|
||||
base_dir = self.get_argument('path')
|
||||
if (base_dir==None or uid==None or gid==None):
|
||||
self.write("Invalid argument!You caused a %d error."%status_code)
|
||||
exit(1)
|
||||
if(os.path.exists(base_dir)):
|
||||
statinfo = os.stat(base_dir)
|
||||
self.write('{'+'"father_node"'+':')
|
||||
statdict = {'path':base_dir,'mode':str(statinfo.st_mode),'ino':str(statinfo.st_ino),'dev':str(statinfo.st_dev),'nlink':str(statinfo.st_nlink),'uid':str(statinfo.st_uid),'gid':str(statinfo.st_gid),'size':str(statinfo.st_size),'atime':str(statinfo.st_atime),'mtime':str(statinfo.st_mtime),'ctime':str(statinfo.st_ctime)}
|
||||
if(int(uid)==statinfo.st_uid and int(gid)==statinfo.st_gid):
|
||||
self.write(statdict)
|
||||
mode = statinfo.st_mode
|
||||
else:
|
||||
self.write("Permission denied.")
|
||||
exit(1)
|
||||
else:
|
||||
self.write("File or directory doesn't exist!You caused a %d error."%status_code)
|
||||
exit(1)
|
||||
if (S_ISDIR(mode)==None):
|
||||
self.write("This is not a directory!You caused a %d error."%status_code)
|
||||
exit(1)
|
||||
else:
|
||||
files = os.listdir(base_dir)
|
||||
for f in files:
|
||||
statinfo = os.stat(base_dir + '/' +f)
|
||||
self.write(',"'+f+'":')
|
||||
statdict = {'path':(base_dir + '/' +f),'mode':str(statinfo.st_mode),'ino':str(statinfo.st_ino),'dev':str(statinfo.st_dev),'nlink':str(statinfo.st_nlink),'uid':str(statinfo.st_uid),'gid':str(statinfo.st_gid),'size':str(statinfo.st_size),'atime':str(statinfo.st_atime),'mtime':str(statinfo.st_mtime),'ctime':str(statinfo.st_ctime)}
|
||||
# print ("statdict",statdict)
|
||||
self.write(statdict)
|
||||
self.write("}")
|
||||
def write_error(self,status_code,**kwargs):
|
||||
self.write("Gosh darnit,user!You caused a %d error."%status_code)
|
||||
|
||||
class StreamingRequestHandler(tornado.web.RequestHandler):
|
||||
@tornado.web.asynchronous
|
||||
@gen.coroutine
|
||||
def get(self):
|
||||
# back as right now this is limited to what is hard coded in
|
||||
total_sent = 0
|
||||
uid = self.get_argument('uid')
|
||||
gid = self.get_argument('gid')
|
||||
base_dir = self.get_argument('filepath')
|
||||
if (base_dir==None or uid==None or gid==None):
|
||||
self.write("Invalid argument!You caused a %d error."%status_code)
|
||||
exit(1)
|
||||
if(os.path.exists(base_dir)):
|
||||
statinfo = os.stat(base_dir)
|
||||
if(int(uid)==statinfo.st_uid and int(gid)==statinfo.st_gid):
|
||||
mode = statinfo.st_mode
|
||||
else:
|
||||
self.write("Permission denied.")
|
||||
exit(1)
|
||||
else:
|
||||
self.write("File or directory doesn't exist!You caused a %d error."%status_code)
|
||||
exit(1)
|
||||
if (S_ISDIR(mode)):
|
||||
self.write("This is not a file!You caused a %d error."%status_code)
|
||||
exit(1)
|
||||
else:
|
||||
with open(base_dir, 'rb') as infile:
|
||||
for chunk in read_in_chunks(infile):
|
||||
self.write(chunk)
|
||||
yield gen.Task(self.flush)
|
||||
total_sent += len(chunk)
|
||||
print("sent",total_sent)
|
||||
|
||||
self.finish()
|
||||
|
||||
if __name__ == "__main__":
|
||||
# this was connected to the pyCurl call and as far as I know now not
|
||||
# beng used so try without to insure it's no longer needed
|
||||
# tornado.options.parse_command_line()
|
||||
print (tornado.version)
|
||||
application = tornado.web.Application([
|
||||
(r"/download", StreamingRequestHandler),
|
||||
(r"/list",ListRequestHandler),
|
||||
(r"/read",ReadRequestHandler)
|
||||
])
|
||||
application.listen(8880)
|
||||
tornado.ioloop.IOLoop.instance().start()
|
||||
|
||||
Loading…
Reference in New Issue