如何使用subprocess从zcat读取多个stdin

Question

我想把下面这个shell命令转换成Python代码，使用subprocess模块。特别是，怎么把多个<(zcat ...)作为标准输入（stdin）呢？理想情况下，代码应该使用subprocess.call，不过subprocess.Popen也可以。

rsem-calculate-expression \
-p 2 \
--some-other-option \
--paired-end \
<(zcat a_1.fastq.gz b_1.fastq.gz c_1.fastq.gz) \
<(zcat b_2.fastq.gz b_2.fastq.gz c_2.fastq.gz) \
~/index/hg19 \
output_dir \
1>stdin.log \
2>stderr.log

Answer 1

另一种运行bash命令的方法是使用 shell=True：

from subprocess import check_call

check_call('rsem-calculate-expression -p 2 --some-other-option '
           '--paired-end '
           '<(zcat a_1.fastq.gz b_1.fastq.gz c_1.fastq.gz) '
           '<(zcat b_2.fastq.gz b_2.fastq.gz c_2.fastq.gz) '
           '~/index/hg19 '
           'output_dir '
           '1>stdin.log '
           '2>stderr.log', shell=True, executable='/bin/bash')

为了对比，这里是如何在不使用shell的情况下运行命令：

#!/usr/bin/env python3
import os
import shlex
from contextlib import ExitStack # $ pip install contextlib2 on Python 2
from shutil import rmtree
from subprocess import Popen
from tempfile import mkdtemp

# convert command-line into a list (program arguments)
args = 'rsem-calculate-expression -p2 --some-other-option --paired-end'.split()
zcat_args = [shlex.split('zcat a_1.fastq.gz b_1.fastq.gz c_1.fastq.gz'),
             shlex.split('zcat b_2.fastq.gz b_2.fastq.gz c_2.fastq.gz')]
npipes = len(zcat_args)
with ExitStack() as stack:
    # create named pipes
    pipenames = []
    dirname = mkdtemp() # create temporary directory for named pipes
    stack.callback(rmtree, dirname) # clean up at the end
    for i in range(npipes):
        pipename = os.path.join(dirname, 'zcat_named_pipe' + str(i))
        os.mkfifo(pipename)
        args.append(pipename) # append to the program args
        pipenames.append(pipename)

    # run rsem-calculate-expression that reads from the named pipes
    args.append(os.path.expanduser('~/index/hg19'))
    args.append('output_dir')
    with open('stdout.log', 'wb', 0) as f1, open('stderr.log', 'wb', 0) as f2:
        rsem = Popen(args, stdout=f1, stderr=f2)
    stack.callback(rsem.wait)

    # run zcat commands to populate the named pipes
    for pipename, cmd in zip(pipenames, zcat_args):
        if rsem.poll() is None: # it is still running and reading from pipes
            with open(pipename, 'wb', 0) as pipe: # may block if no readers
                stack.callback(Popen(cmd, stdout=pipe).wait)
exit(rsem.returncode != 0)

Answer 2

最简单的方法就是让bash来处理这个过程替换：

subprocess.Popen(['/bin/bash', '-c', 'rsem-calculate-expression -p 2 \
    --some-other-option --paired-end \
    <(zcat a_1.fastq.gz b_1.fastq.gz c_1.fastq.gz) \
    <(zcat b_2.fastq.gz b_2.fastq.gz c_2.fastq.gz) \
    ~/index/hg19 output_dir'])