如何对包含多处理的函数进行多处理

Question

我在网上的某个地方找到了一个非常复杂的函数。我确信该函数本身使用 python 多处理包。

现在我有很多系列的数据，我还想在该功能中使用多处理。

以下是该函数的简化版本：

import multiprocessing
import numpy as np
def main(number=10):
    print('ok')
    def segment():
        result=[]
        error='meet bad case'
        for i in range(number):
            result.append(i*np.random.rand())
            if result[-1] ==number/2:
                raise AssertionError(error)
        json.dump(result, codecs.open(os.getcwd()+'/data/result'+str(number), 
                'w', encoding='utf-8'), separators=(',', ':'), sort_keys=True, indent=4) 
    p = multiprocessing.Process(target=segment)
    p.daemon = True
    p.start()
    p.join()  

这个主函数代表我下载的复杂函数。现在，如何使用平行多处理技术将此功能应用于从 10 到 100 的可变数字范围？

Answer 1

#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import multiprocessing
# We must import this explicitly, it is not imported by the top-level
# multiprocessing module.
import multiprocessing.pool
import time
from random import randint

class NoDaemonProcess(multiprocessing.Process):
    # make 'daemon' attribute always return False
    def _get_daemon(self):
        return False
    def _set_daemon(self, value):
        pass
    daemon = property(_get_daemon, _set_daemon)
# We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool
# because the latter is only a wrapper function, not a proper class.
class MyPool(multiprocessing.pool.Pool):
    Process = NoDaemonProcess
def main(number):
    def segment():
        result=[]
        error='meet bad case'
        for i in range(number):
            result.append(i*np.random.rand())
            if result[-1] ==number/2:
                raise AssertionError(error)
        json.dump(result, codecs.open(os.getcwd()+'/data/result'+str(number), 
                'w', encoding='utf-8'), separators=(',', ':'), sort_keys=True, indent=4) 
    p = multiprocessing.Process(target=segment)
    p.start()
    p.join() 
    print('ok')
def test():
    print("Creating 5 (non-daemon) workers and jobs in main process.")
    pool = MyPool(5)
    pool.map(main, [5,6,7])
    pool.close()
    pool.join()
if __name__ == '__main__':
    test()