添加记录器原因无法pickle_螺纹锁紧物体

Question

这是我的密码

class MusicHandler(object): """ Implements the logic to download musics """ def __init__(self): """ Creates the class object, loads configs, absolute paths and create directories. Does not init download """ # create logger self.logger = getLogger("music logger") self.logger.info("Initializing MusicHandler class object") # load config vars self.configs = utils.get_configs() # load absolute paths self.dir_music = self.configs["music_path"] self.dir_audio = utils.get_path_from_rel(self.configs["audio_path"], base_path=self.dir_music) self.dir_video = utils.get_path_from_rel(self.configs["video_path"], base_path=self.dir_music) self.taboo_path = utils.get_path_from_rel(self.configs["taboo_path"]) # make dir if not exists list(map(utils.make_dir_safe, (self.dir_music, self.dir_audio, self.dir_video))) @run_safe def download_one(self, link, keep_video=False): """ handles the download of one link. keep_video determines if video is saved or deleted. """ self.logger.info(f"Starting download of {link}") if self.is_taboo(link): self.logger.warning("Link is taboo, will skip it.") return name = self.download_video(link) self.logger.info(f"Download of {name} (video) was a success.") self.video_to_audio(name) self.add_to_taboo(link) if not keep_video: self.remove_video(name) self.logger.critical(f"Successfully downloaded {name}. Available at {self.dir_music}.") def get_musics_linear(self, url_list): """ Linear download of all links from list """ for link in url_list: self.download_one(link) @run_safe def get_musics_parallel(self, url_list, num_procs=3): """ Parallel download of all links from list """ with Pool(num_procs) as p: print("here") p.map(self.download_one, url_list) print("there") def is_taboo(self, link): """ Verifies if link exists in the taboo file """ return utils.is_in_file(self.taboo_path, link) def add_to_taboo(self, link): """ Adds link to the taboo file """ utils.append_to_file(self.taboo_path, link) def download_video(self, link): """ Downloads the highest resolution video given a link. Returns file name """ yt = YouTube(link) yt.streams.filter(progressive=True).order_by("resolution").desc().first().download(self.dir_video, yt.title) return utils.safe_filename(yt.title) def download_audio(self, link): """ Downloads only the audio of a given link. Returns file name """ yt = YouTube(link) yt.streams.filter(only_audio=True).first().download(self.dir_audio, yt.title) return utils.safe_filename(yt.title) def video_to_audio(self, name): """ Converts a video to an audio file """ self.logger.info("Converting video to audio.") video_f_path = os.path.join(self.dir_video, "{}{}".format(name, ".mp4")) audio_f_path = os.path.join(self.dir_audio, "{}{}".format(name, ".wav")) cmd_list = ["ffmpeg", "-i", video_f_path, "-f", "wav", "-ab", "19200", "-vn", audio_f_path] FNULL = open(os.devnull, 'w') subprocess.call(cmd_list, stdout=FNULL, stderr=subprocess.STDOUT) def remove_video(self, name): """ Deletes a video file """ self.logger.info("Removing video file.") video_f_path = os.path.join(self.dir_video, "{}{}".format(name, ".mp4")) if os.path.exists(video_f_path): os.remove(video_f_path)

Answer 1

我在运行pyspark作业时遇到了这个问题，在该作业中，我使用日志模块记录任务期间的信息。
火花日志显示了这样的信息”_pickle.pickling错误错误：无法序列化对象：无法_螺纹锁紧对象“
Spark会在计算之前将变量强制转换到所有的工人，但是日志对象不能被强制转换，这导致了这样的错误。
删除所有日志记录后，错误消失。
希望这个提示对你有帮助。在

Answer 2

您不是在某处使用multiprocessing模块吗？它要求线程/进程之间交换的数据是可选择的。在

记录器使用^{}，它保存进程的状态，因此不能进行pickle。Logger uses it这样，当整个脚本中有多个记录器（或同一个）被获取时，它就可以实现不弄乱日志文件（或任何其他日志输出）的魔力。在