如何在for循环中进行多进程

Question

import string
import hashlib
import itertools
hash_list = [hash1,hash2,hash3,...]
allchar = string.ascii_letters + string.digits + string.punctuation
def crack(msg):
    temp = hashlib.md5(msg.encode('utf-8)).hexdigest()
    for hashvalue in hash_list:
        if temp == hashvalue:
            print(msg)
            hash_list.remove(hashvalue)
if __name__ == '__main__':
    n = 1
    while len(hash_list) > 0:
        for result in itertools.product(allchar, repeat = n):
            msg = ''.join(result)
            crack(msg)
        n += 1

我写了这个用于开裂哈希的程序。如何在此程序中使用多处理？我试图将Itertool的所有产品存储在列表中以进行映射，但是该列表变得太大了。是否有一种记忆有效的方法来处理此程序？非常感谢！

Answer 1

您可以从concurrent.futures中使用ProcessPoolExecutor。

from concurrent import futures
from multiprocess import Manager
hash_list = Manager().list([hash1, hash2, hash3, ...])
def crack(msg):
    temp = hashlib.md5(msg.encode("utf-8")).hexdigest()
    for hashvalue in hash_list:
        if temp == hashvalue:
            print(msg)
            hash_list.remove(hashvalue)
if __name__ == "__main__":
    n = 1
    while hash_list;
        with futures.ProcessPoolExecutor() as executor:
            executor.map(crak, [''.join(result) for result in itertools.product(allchar, repeat = n)])
        n += 1

Answer 2

您是否尝试过生成itertools.product的子集，然后将每个子集传递给一个工人：

from multiprocessing import Pool
def crack(alist):
    ...
    return result
def gather(result):
    ...
if __name__ == '__main__':
    pool = Pool()
    g = itertools.product(allchar, repeat = n)
    while True:
        try:
            ss = [next(g) for _ in _ in range(10000)]
            pool.apply_async(crack, ss, gather)
        except StopIteration:
            pool.apply_async(crack, ss, gather)
            break

编辑per @juanpa.arrivillaga，并从：https：//stackoverflow.com/a/43079667/2718295

设置一些队列并在主过程中填充它。工人负责消耗密码：

from multiprocessing import Queue, Pool, Lock
def crack(q, q2, lock):
    while True:
        temp = q.get()
        if temp is None:
            break
        hash_list = q2.get()
        if not hash_list:
            # empty set
            break
        h = hashlib.md5(temp.encode('utf-8')).hexdigest()
        for hashvalue in hash_list:
            if h == hashvalue:
                hash_list = hash_list.discard(hashvalue)
                with lock:
                    print(''.join(temp))
                    # resynchronize the hashlist
                    hash_list2 = q2.get()
                    q2.put(hash_list & hash_list2 )     

if __name__ == '__main__':
    NCORE = 4
    q = Queue(maxsize=NCORE)
    q2 = Queue(maxsize=NCORE)
    lock = Lock()
    hash_list = set([hash1,hash2,hash3,...])
    allchar = string.ascii_letters + string.digits + string.punctuation
    pool = Pool(NCORE, initializer=crack, initargs=(q,q2,lock))
    q2.put(hash_list)
    for result in itertools.product(allchar, repeat = n):
        q.put(result)
    for _ in range(NCORE):
        q.put(None)
    pool.close()
    pool.join()