如何使下面的函数线程化

Question

我对python和编程都很陌生-寻找关于如何收紧下面的函数并节省一些时间的建议。一些背景信息:

要求是我收集给定顶级文件夹中每个子文件/文件夹的名称和ID。问题是，我请求数据的服务器将只返回单个文件夹的内容，并且响应将始终指定返回的对象是文件还是文件夹。

(伪代码示例，只是试图快速演示):

Top_level_folderid = 1111
url = "fileserverapi.couldbebetter.com/thismighttakeawhile"
post(url, data=Top_level_folderid)
response({"jim's folder" : id=1234, filetype=folder}, {"weird_video.mp4" : id=4321, filetype=file}) 

然后我必须遍历每个响应并返回到服务器以获得下一组，在某些情况下，整个顶级文件夹可能包含多达15,000个文件夹和30,000多个随机分布的文件，其中一些文件夹包含1个文件和15个文件夹，其他文件夹包含7个文件和2个子文件夹等。

API本身响应相当快，但是我不知道它可以处理多少并发连接，所以我需要能够调整并找到函数内的最佳点，猜测它将处理10-50的任何地方。我现在的功能:

def drill_folder_loop(folder_list, project_id, directory, jsession_id):
    count = 0
    temp_folder_list = folder_list #\ A list of dicts [{folder_name1 : folder_id1}, {folder_name2 : folder_id2}]
    while count == 0:
        temp_folder_list1 = []
        for folder in temp_folder_list:
            f_name = folder.keys()[0] #// folder_name (not actually used here)
            f_id = folder.values()[0] #// folder id
            folder_dict = list_folders_files(f_id, jsession_id) #// list_folders_files posts to the api and builds the response back into a list of dicts, same as the original passed to this function.
            folder_list = process_folder_files(folder_dict, directory, jsession_id) #// somewhat irrelevant to the question - I have to commit the file data to a DB, I could strip the folder list in this function but since i need it there I just return it instead.
            process_recipients = recipient_process(folder_dict, "no", f_id,
                                                   directory, project_id)#// more irrelevant but necessary processing.
            for i in range(0, len(folder_list)):
                append_f = folder_list[i]
                temp_folder_list1.append(append_f)#// append new folders to list outside loop
        temp_folder_list = [] #// empty temp_folder_list, loop may contain more than a single folder so I have to empty it once I've processed all the folders
        for i in range(0, len(temp_folder_list1)):#// Put the new folder list into temp_folder_list so the loop restarts
            append_f2 = temp_folder_list1[i]
            temp_folder_list.append(append_f2)
        if not temp_folder_list: #// end the loop if there are no more sub-folders
            count += 1
    return log_info("Folder loop complete")

重读这是变量命名的一个很好的教训…不是最简洁的…代码本身工作得很好，但正如你现在可能想象的那样，在数千个文件夹中翻找需要很长时间……关于如何将其变成多线程/处理野兽的任何建议/方向?感谢您花时间阅读这篇文章!

编辑:

为了清楚起见，我不想在循环中处理文件夹，而是想在线程中任务它们，以便有多个文件夹，因此同时发生的post请求和响应，以便整个过程花费更少的时间。现在它只是一次循环一个文件夹。希望这能澄清…

编辑:

从Noctis Skytower的建议中，我做了一些小的改变来支持python 2.7 (Queue vs Queue和。clock()而不是perf_counter())。太近了!我遇到的问题是，当我将运行线程更改为1时，它完美地完成-当我出于某种原因(随机)将其增加回25时，dfl_worker()中的变量f_id为None。考虑到它可以与1线程一起工作，我猜这不是建议的问题，而是我代码中的其他东西，所以我将其标记为接受。谢谢!

class ThreadPool:
    def __init__(self, count, timeout, daemonic):
        self.__busy = 0
        self.__idle = clock()
        self.__jobs = Queue()
        self.__lock = Lock()
        self.__pool = []
        self.__timeout = timeout
        for _ in range(count):
            thread = Thread(target=self.__worker)
            thread.daemon = daemonic
            thread.start()
            self.__pool.append(thread)
    def __worker(self):
        while True:
            try:
                function, args, kwargs = self.__jobs.get(True, 0.1)
            except Empty:
                with self.__lock:
                    if self.__busy:
                        continue
                    if clock() - self.__idle < self.__timeout:
                        continue
                    break
            else:
                with self.__lock:
                    self.__busy += 1
                try:
                    function(*args, **kwargs)
                except:
                    pass
                with self.__lock:
                    self.__busy -= 1
                    self.__idle = clock()
    def apply(self, function, *args, **kwargs):
        self.__pool = list(filter(Thread.is_alive, self.__pool))
        if not self.__pool:
            raise RuntimeError('ThreadPool has no running Threads')
        self.__jobs.put((function, args, kwargs))
    def join(self):
        for thread in self.__pool:
            thread.join()

def drill_folder_loop(folder_list, project_id, directory, jsession_id):
    tp = ThreadPool(25, 1, False)
    tp.apply(dfl_worker, tp, folder_list, project_id, directory, jsession_id)
    tp.join()

def dfl_worker(tp, folder_list, project_id, directory, jsession_id):
    for folder in folder_list:
        f_name = folder.keys()[0]
        f_id = folder.values()[0]
        f_dict = list_folders_files(f_id, jsession_id)
        f_list = process_folder_files(f_dict, directory, jsession_id)
        tp.apply(dfl_worker, tp, f_list, project_id, directory, jsession_id)
        recipient_process(f_dict, 'no', f_id, directory, project_id)
    log_info('One folder processed')

Answer 1

我可以推荐以下产品吗?

from queue import Empty, Queue
from threading import Lock, Thread
from time import perf_counter

def drill_folder_loop(folder_list, project_id, directory, jsession_id):
    while True:
        next_folder_list = []
        for folder in folder_list:
            f_name, f_id = folder.popitem()
            f_dict = list_folders_files(f_id, jsession_id)
            f_list = process_folder_files(f_dict, directory, jsession_id)
            recipient_process(f_dict, 'no', f_id, directory, project_id)
            next_folder_list.extend(f_list)
        if not next_folder_list:
            break
        folder_list = next_folder_list
    return log_info('Folder loop complete')
###############################################################################

class ThreadPool:
    def __init__(self, count, timeout, daemonic):
        self.__busy = 0
        self.__idle = perf_counter()
        self.__jobs = Queue()
        self.__lock = Lock()
        self.__pool = []
        self.__timeout = timeout
        for _ in range(count):
            thread = Thread(target=self.__worker)
            thread.daemon = daemonic
            thread.start()
            self.__pool.append(thread)
    def __worker(self):
        while True:
            try:
                function, args, kwargs = self.__jobs.get(True, 0.1)
            except Empty:
                with self.__lock:
                    if self.__busy:
                        continue
                    if perf_counter() - self.__idle < self.__timeout:
                        continue
                    break
            else:
                with self.__lock:
                    self.__busy += 1
                try:
                    function(*args, **kwargs)
                except:
                    pass
                with self.__lock:
                    self.__busy -= 1
                    self.__idle = perf_counter()
    def apply(self, function, *args, **kwargs):
        self.__pool = list(filter(Thread.is_alive, self.__pool))
        if not self.__pool:
            raise RuntimeError('ThreadPool has no running Threads')
        self.__jobs.put((function, args, kwargs))
    def join(self):
        for thread in self.__pool:
            thread.join()

def drill_folder_loop(folder_list, project_id, directory, jsession_id):
    tp = ThreadPool(25, 1, False)
    tp.apply(dfl_worker, tp, folder_list, project_id, directory, jsession_id)
    tp.join()

def dfl_worker(tp, folder_list, project_id, directory, jsession_id):
    for folder in folder_list:
        f_name, f_id = folder.popitem()
        f_dict = list_folders_files(f_id, jsession_id)
        f_list = process_folder_files(f_dict, directory, jsession_id)
        tp.apply(dfl_worker, tp, f_list, project_id, directory, jsession_id)
        recipient_process(f_dict, 'no', f_id, directory, project_id)
    log_info('One folder processed')

第一个drill_folder_loop是你的函数的重写，应该做同样的事情，但第二个版本应该利用ThreadPool类，这样你的文件夹列表可以由多达25个线程并发处理。请注意，线程版本不会返回任何有意义的内容，如果在末尾删除tp.join()，则执行后几乎立即返回。

Answer 2

要理解你想用这段代码做什么有点棘手。

据我所知，你想让这个代码多线程;要做到这一点，你需要找到一个可以相互独立执行的routine/taskset，然后你可以把它从循环中取出来，并创建一个独立的函数。

def task(someparams):
    #mytasks using the required someparams

现在您可以创建一组工作线程，并为它们分配工作以运行task例程并完成您的工作。

下面是task例程的多线程方法:

import thread
WORKER_THREAD = 10
c = 0
while c < WORKER_THREAD: 
    thread.start_new_thread( task, (someparams, ) )
while 1:
   pass

线程。Start_new_thread (task， (someparams，))