通过Python客户端将大文件流式POST到CherryPy

Question

我想从一个Python客户端向cherrypy发送一个大文件。我正在使用requests库。

这是我的客户端代码：

def upload(fileName=None):
    url = 'http://localhost:8080/upload'
    files = {'myFile': ( fileName, open(fileName, 'rb') )}
    r = requests.post(url, files=files)

#with open(fileName,'rb') as payload:
    #headers = {'content-type': 'multipart/form-data'}
    #r = requests.post('http://127.0.0.1:8080', data=payload,verify=False,headers=headers)

if __name__ == '__main__':
    upload(sys.argv[1])

问题是，这样会把整个文件都放到内存里。有没有办法把文件分成几块来发送呢？

class FileDemo(object):


@cherrypy.expose
def upload(self, myFile):

    print myFile.filename
    #size = 0
    #decoder = MultipartDecoder(myFile, 'image/jpeg')
    #for part in decoder.parts:
        #print(part.header['content-type'])

    #while True:

        #advances to the content that hasn't been read
        #myFile.file.seek(size, 0)

        #reads 100mb at a time so it doesn't fill up the RAM
        #data = myFile.file.read(10240000)

        #newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
        #newFile.write(data)
        #newFile.close

        #size += len(data)

        #if len(data) < 10240000:
            #break
if __name__ == '__main__':
    cherrypy.quickstart(FileDemo())

这是服务器端的代码。里面有很多注释，因为我尝试了很多方法。现在我只是打印文件名，而客户端仍然把整个文件传输到内存里。

我不知道还可以尝试什么。提前谢谢你的帮助。

Answer 1

这是我解决这个问题的方法：

客户端

import poster
def upload(fileName=None):

    register_openers()
    url = 'http://localhost:8080/upload'
    data, headers = multipart_encode({"myFile": open(fileName, "rb")})

    request = urllib2.Request(url, data, headers)
    request.unverifiable = True
    response = urllib2.urlopen(request)
    the_page = response.read()


if __name__ == '__main__':
    upload(sys.argv[1])

服务器

@cherrypy.expose
def upload(self, myFile):

    cherrypy.response.timeout = 3600
    newFile = open("/home/ivo/Desktop/"+str(myFile.filename), 'a+')
    newFile.write(myFile.file.read())
    newFile.close

Answer 2

如果你是在使用CherryPy进行文件上传的话，可以省去一些复杂的编码步骤，比如 multipart/form-data，直接发送文件内容的流式POST请求就可以了。

客户端

#!/usr/bin/env python
# -*- coding: utf-8 -*-


import urllib2
import io
import os


class FileLenIO(io.FileIO):

  def __init__(self, name, mode = 'r', closefd = True):
    io.FileIO.__init__(self, name, mode, closefd)

    self.__size = statinfo = os.stat(name).st_size

  def __len__(self):
    return self.__size


f = FileLenIO('/home/user/Videos/video.mp4', 'rb')
request = urllib2.Request('http://127.0.0.1:8080/upload', f)
request.add_header('Content-Type', 'application/octet-stream')
# you can add custom header with filename if you need it
response = urllib2.urlopen(request)

print response.read()

服务器

#!/usr/bin/env python
# -*- coding: utf-8 -*-


import os
import tempfile
import shutil

import cherrypy


config = {
  'global' : {
    'server.socket_host' : '127.0.0.1',
    'server.socket_port' : 8080,
    'server.thread_pool' : 8,
    # remove any limit on the request body size; cherrypy's default is 100MB
    'server.max_request_body_size' : 0,
    # increase server socket timeout to 60s; cherrypy's defult is 10s
    'server.socket_timeout' : 60
  }
}


class App:

  @cherrypy.config(**{'response.timeout': 3600}) # default is 300s
  @cherrypy.expose()
  def upload(self):
    '''Handle non-multipart upload'''

    destination = os.path.join('/home/user/test-upload')                
    with open(destination, 'wb') as f:
      shutil.copyfileobj(cherrypy.request.body, f)

    return 'Okay'


if __name__ == '__main__':
  cherrypy.quickstart(App(), '/', config)

在测试一个1.3GB的视频文件时，服务器的内存使用量不到10MB，而客户端的内存使用量不到5MB。