如何找出“无法pickle”\u io.BufferedReader“object”错误

2024-04-28 04:48:31 发布

您现在位置:Python中文网/ 问答频道 /正文

我试图通过多重处理下载插图。这几天我一直在为多重处理而挣扎。请帮我解决这个问题

这里我的代码如下

from urllib.request import urlopen
from bs4 import BeautifulSoup
import urllib
import multiprocessing
import sys
from multiprocessing import Lock
import time
import pickle


path = 'D:/my_folder/illust'
num = 0
b = 0
tf = False


def get_links(numb):
    time.sleep(1)
    url = 'https://anime-pictures.net/pictures/view_posts/' + str(numb) + '?lang=en'
    html = urlopen(url)
    bsObject = BeautifulSoup(html, "html.parser")
    urls = bsObject.find_all("span", class_="img_block_big")
    print(len(urls))
    return urls

def get_content(urls):
    global b
    imgSiteUrl = urls.find("a")["href"]
    print(imgSiteUrl)
    url_ = 'https://anime-pictures.net' + imgSiteUrl
    html_ = urlopen(url_)
    bsObject_ = BeautifulSoup(html_, "html.parser")
    urls_ = bsObject_.find("div", id="big_preview_cont")
    imgSiteUrl_ = urls_.find("a")["href"]
    imgurl = 'https://anime-pictures.net' + imgSiteUrl_
    b += 1
    print(b)
    print(imgurl)
    #full_url = path + "/" + "test" + str(imgSiteUrl_.split('/')[3].split('-')[0]) + ".jpg"
    #urllib.request.urlretrieve(imgurl, full_url)

if __name__ == '__main__':
    while True:
        start_time = time.time()
        sys.setrecursionlimit(20000)
        pool = multiprocessing.Pool(processes=8)
        pool.map(get_content, get_links(0), chunksize=1)
        num += 1
        print(num)
        print('time' + str(time.time() - start_time))

当我运行这个程序时,我得到下面这个错误

Traceback (most recent call last):
  File "D:/my_flolder/python/test.py", line 48, in <module>
    pool.map(get_content, get_links(0), chunksize=1)
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.1776.0_x64__qbz5n2kfra8p0\lib\multiprocessing\pool.py", line 364, in map
    return self._map_async(func, iterable, mapstar, chunksize).get()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.1776.0_x64__qbz5n2kfra8p0\lib\multiprocessing\pool.py", line 771, in get
    raise self._value
multiprocessing.pool.MaybeEncodingError: Error sending result: '<multiprocessing.pool.ExceptionWithTraceback object at 0x000001DA4B722100>'. Reason: 'TypeError("cannot pickle '_io.BufferedReader' object")'

错误是什么意思?我如何修复它


Tags: fromimporturlmapgettimehtmlfind