Python reads with fd right in bytearray

Is there a means to read from a file descriptor (and not an object like IO) directly in bytearray ?

I am currently using a temporary FileIO object for notification, for example:

 def fd_readinto(fd, ba): fio = io.FileIO(fd, closefd = False) return fio.readinto(ba) 
+4
source share
1 answer

There is no function for this, and your method is already the fastest.

I was going to suggest bytearray(mmap) , array.fromfile and even homegrown os.read() using bytearray and memoryview , but FileIO.readinto screams quickly. (This makes sense because it only makes one system call.)

 import os import mmap, io, array import timeit fn = 'path-to-largeish-file' def fd_readinto_mmap(fd, ba): m = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) ba.extend(m) m.close() def fd_readinto_fio(fd, ba): sz = os.fstat(fd).st_size ba2 = bytearray(sz) with io.FileIO(fd, closefd = False) as fio: fio.readinto(ba2) ba.extend(ba2) def fd_readinto_array(fd, ba): ar = array.array('c') sz = os.fstat(fd).st_size fp = os.fdopen(fd, 'rb') ar.fromfile(fp, sz) ba.extend(ar) def fd_readinto_mv(fd, ba): stat = os.fstat(fd) blksize = getattr(stat, 'st_blksize', 4096) bufsize = stat.st_size buf = bytearray(bufsize) m = memoryview(buf) while True: b = os.read(fd, blksize) s = len(b) if not s: break m[:s], m = b, m[s:] writtenbytes = buffer(buf, 0, bufsize-len(m)) ba.extend(writtenbytes) setup = """ from __main__ import fn, fd_readinto_mmap, fd_readinto_fio, fd_readinto_array, fd_readinto_mv import os openfd = lambda : os.open(fn, os.O_RDONLY) closefd = lambda fd: os.close(fd) """ reps = 2 tests = { 'fio' : "fd=openfd(); fd_readinto_fio(fd, bytearray()); closefd(fd)", 'mmap': "fd=openfd(); fd_readinto_mmap(fd, bytearray()); closefd(fd)", 'array': "fd=openfd(); fd_readinto_array(fd, bytearray());", 'mv' : "fd=openfd(); fd_readinto_mv(fd, bytearray()); closefd(fd)", } width = max(map(len, tests)) for n,t in tests.iteritems(): time = timeit.timeit(t, setup, number=reps) print ("{:%s} {}" % width).format(n, time) 

On my system (OS X 10.14.6, Python 2.7.10), FileIO is the fastest option:

 mmap 7.19839119911 array 5.72453403473 mv 0.49933886528 fio 0.299485206604 
0
source

All Articles