多处理---重新启动进程逻辑

Question

在多处理中，启动了一个新进程，但它就像在每个循环中一样吗？ 例如 - 我正在多个页面(200 页(上进行图像处理，因此在每个页面中都会启动一个新过程。或者，如果有 4 核处理器，那么在一个循环中，python 的八个进程将启动，然后在下一个循环中，该过程将继续而不重新启动 python？或者纠正我，如果我的基础知识弄错了......

Answer 1

如果要对多个图像执行相同的操作，使用concurrent.futures.ProcessPoolExecutor将为您处理大部分细节。您只需编写函数即可处理图像。

我建议使用concurrent.futures.as_completed()方法或Executor.map()方法。

如果要通过使用subprocess调用外部程序来处理图像，请改用ThreadPoolExecutor。下面是一个完整的示例，可以将TIFF文件转换为PDF。 您可以在我的脚本存储库中找到此脚本和其他几个类似的脚本。

(请注意，ProcessPoolExecutor可以以与ThreadPoolExecutor相同的方式使用，因为它们都继承自Executor类。

from functools import partial
import argparse
import concurrent.futures as cf
import logging
import os
import re
import subprocess as sp
import sys
__version__ = '1.4'

def main():
"""
Entry point for tifftopdf.
"""
args = setup()
func = tiffconv
if args.jpeg:
logging.info('using JPEG compression.')
func = partial(tiffconv, jpeg=True, quality=args.quality)
with cf.ThreadPoolExecutor(max_workers=os.cpu_count()) as tp:
for fn, rv in tp.map(func, args.files):
if rv == 0:
logging.info(f'finished "{fn}"')
else:
logging.error(f'conversion of {fn} failed, return code {rv}')

def setup():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
'--log',
default='warning',
choices=['debug', 'info', 'warning', 'error'],
help="logging level (defaults to 'warning')"
)
parser.add_argument('-j', '--jpeg', help='use JPEG compresion', action="store_true")
parser.add_argument(
'-q', '--quality', help='JPEG compresion quality (default 85)', type=int, default=85
)
parser.add_argument('-v', '--version', action='version', version=__version__)
parser.add_argument("files", metavar='file', nargs='+', help="one or more files to process")
args = parser.parse_args(sys.argv[1:])
logging.basicConfig(
level=getattr(logging, args.log.upper(), None), format='%(levelname)s: %(message)s'
)
logging.debug(f'command line arguments = {sys.argv}')
logging.debug(f'parsed arguments = {args}')
# Check for requisites
try:
for prog in ('tiffinfo', 'tiff2pdf'):
sp.run([prog], stdout=sp.DEVNULL, stderr=sp.DEVNULL)
logging.debug(f'found “{prog}”')
except FileNotFoundError:
logging.error(f'required program “{prog}” not found')
sys.exit(1)
return args

def tiffconv(fname, jpeg=False, quality=85):
"""
Start a tiff2pdf process for given file.
Arguments:
name: Name of the tiff file to convert.
jpeg: Use JPEG compression.
quality: JPEG compression quality.
Returns:
A 2-tuple (input filename, tiff2pdf return value).
"""
try:
args = ['tiffinfo', fname]
p = sp.run(args, stdout=sp.PIPE, stderr=sp.DEVNULL)
txt = p.stdout.decode().split()
if 'Width:' not in txt:
raise ValueError('no width in TIF')
index = txt.index('Width:')
width = float(txt[index + 1])
length = float(txt[index + 4])
try:
index = txt.index('Resolution:')
xres = float(txt[index + 1][:-1])
yres = float(txt[index + 2])
except ValueError:
xres, yres = None, None
outname = re.sub(r'.tif{1,2}?$', '.pdf', fname, flags=re.IGNORECASE)
program = ['tiff2pdf']
if xres:
args = [
'-w',
str(width / xres), '-l',
str(length / xres), '-x',
str(xres), '-y',
str(yres), '-o', outname, fname
]
else:
args = ['-o', outname, '-z', '-p', 'A4', '-F', fname]
logging.warning(f"no resolution in {fname}. Fitting to A4")
if jpeg:
args = program + ['-n', '-j', '-q', str(quality)] + args
else:
args = program + args
logging.info(f'calling "{args}"')
rv = sp.run(args, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
logging.info(f'created "{outname}"')
return (fname, rv.returncode)
except Exception as e:
logging.error(f'starting conversion of "{fname}" failed: {e}')
return (fname, 0)

if __name__ == '__main__':
main()