python多进程无法获取属性

Question

我有我的职责：

downloadXMLPool = Pool(processes=3)
downloadFile = Pool(processes=3)


def dl_data_and_convert(url,formatIn,retry=True,noHTML=False,downloadSpecialPool=None):
    if downloadSpecialPool == None:
        dlPool = downloadXMLPool
    else:
        dlPool = downloadSpecialPool

    if formatIn == 'xml':
        return dl_xml_to_dict(url,dlPool,retry=retry,noHTML=noHTML)
    elif formatIn == 'json':
        return dl_json_to_dict(url,dlPool,retry=retry,noHTML=noHTML)


def dl_xml_to_dict(url,dlPool,retry=True,tryNumber=1,noHTML=False):
    try:
        response = None
        doc = xmltodict.parse(dlPool.apply_async(requests.get, (url,)).get().content)

        if noHTML:
            if 'html' in doc:
                if retry:
                    sleep(randrange(60,120)*tryNumber)
                    return dl_xml_to_dict(url,dlPool,tryNumber=tryNumber+1,noHTML=noHTML)
        return doc

    except Exception as e:
        if retry and request_error_tester(tryNumber,url,response,e):
            return dl_xml_to_dict(url,dlPool,tryNumber=tryNumber+1,noHTML=noHTML)
        else:
            return None

这两个功能运行良好

我有：

def download_file(url, NameFile, folder='./', retry=True, downloadSpecialPool=None):
    if downloadSpecialPool == None:
        return launch_dl_in_file(url, downloadFile, NameFile, folder=folder, retry=retry)
    else:
        return launch_dl_in_file(url, downloadSpecialPool, NameFile, folder=folder, retry=retry)


def launch_dl_in_file(url, dlPool, NameFile, folder='./', retry=True, tryNumber=1):
    try:
        dlPool.apply_async(dl_in_file, (url, NameFile, folder)).get()
        return True

    except Exception as e:
        if match('.*HTTP Error 504.*',str(e)) != None:
            if retry:
                if tryNumber < 21:
                    sleep(15*tryNumber)
                    return launch_dl_in_file(url, dlPool, NameFile, folder=folder, tryNumber=tryNumber+1)
        print_screen_error("Download error for file : "+url+"\n\tError : "+str(e)+"\n")
        return False


def dl_in_file(url, NameFile, folder='./'):
    with closing(urllib.request.urlopen(url)) as r:
        with open(os.path.join(folder,NameFile), 'wb') as f:
            shutil.copyfileobj(r, f)
    return True

我得到了这个错误：

AttributeError: Can't get attribute 'dl_in_file' on <module 'tools' from '/home/*/src/tools.py'>

我尝试了一些修改：使用另一个多进程池，使用downloadXMLPool，新函数，等等。我仍然有同样的问题

Process ForkPoolWorker-4:
Traceback (most recent call last):
  File "/usr/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/usr/lib/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/lib/python3.7/multiprocessing/pool.py", line 110, in worker
    task = get()
  File "/usr/lib/python3.7/multiprocessing/queues.py", line 354, in get
    return _ForkingPickler.loads(res)
AttributeError: Can't get attribute 'dl_in_file' on <module 'tools' from '/home/*/src/tools.py'>

我可能做错了什么

Answer 1

在Darkonaut的帮助下，我找到了解决办法。 原始线程是：Python Multiprocessing Pool Map: AttributeError: Can't pickle local object

解决方案很简单，您必须移动：

downloadXMLPool = Pool(processes=3)
downloadFile = Pool(processes=3)

在你的功能之后