python从fd直接读入bytearray

Question

有没有一种方法可以直接从文件描述符（而不是类似IO的对象）读入bytearray？在

现在我使用一个临时的FileIO对象进行调解，类似于：

def fd_readinto(fd, ba):
    fio = io.FileIO(fd, closefd = False)
    return fio.readinto(ba)

Answer 1

没有函数可以做到这一点，而您的方法已经是最快的方法了。在

我本来打算建议使用bytearray(mmap)，array.fromfile，甚至是使用bytearray和memoryview自制的os.read()，但是FileIO.readinto很快就会尖叫。（这是有意义的，因为它只执行一个系统调用。）

import os
import mmap, io, array
import timeit

fn = 'path-to-largeish-file'

def fd_readinto_mmap(fd, ba):
    m = mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
    ba.extend(m)
    m.close()

def fd_readinto_fio(fd, ba):
    sz = os.fstat(fd).st_size
    ba2 = bytearray(sz)
    with io.FileIO(fd, closefd = False) as fio:
        fio.readinto(ba2)
    ba.extend(ba2)

def fd_readinto_array(fd, ba):
    ar = array.array('c')
    sz = os.fstat(fd).st_size
    fp = os.fdopen(fd, 'rb')
    ar.fromfile(fp, sz)
    ba.extend(ar)

def fd_readinto_mv(fd, ba):
    stat = os.fstat(fd)
    blksize = getattr(stat, 'st_blksize', 4096)
    bufsize = stat.st_size
    buf = bytearray(bufsize)
    m = memoryview(buf)
    while True:
        b = os.read(fd, blksize)
        s = len(b)
        if not s: break
        m[:s], m = b, m[s:]
    writtenbytes = buffer(buf, 0, bufsize-len(m))
    ba.extend(writtenbytes)

setup = """
from __main__ import fn, fd_readinto_mmap, fd_readinto_fio, fd_readinto_array, fd_readinto_mv
import os
openfd = lambda : os.open(fn, os.O_RDONLY)
closefd = lambda fd: os.close(fd)
"""


reps = 2
tests = {
    'fio' : "fd=openfd(); fd_readinto_fio(fd, bytearray()); closefd(fd)",
    'mmap': "fd=openfd(); fd_readinto_mmap(fd, bytearray()); closefd(fd)",
    'array': "fd=openfd(); fd_readinto_array(fd, bytearray());",
    'mv' : "fd=openfd(); fd_readinto_mv(fd, bytearray()); closefd(fd)",
}

width = max(map(len, tests))
for n,t in tests.iteritems():
    time = timeit.timeit(t, setup, number=reps)
    print ("{:%s} {}" % width).format(n, time)

在我的系统（OS X 10.14.6、Python 2.7.10）上，FileIO是最快的选项：

^{pr2}$