Python zipfile 多线程与进度条结合使用

Question

最近我发现自己越来越常来Stack Overflow了。在学习Python的过程中，我尝试直接把一个C#的应用程序移植过来，这时我遇到了一个之前从未听说过的概念：线程。原本我以为自己对这个概念有了基本的了解，但在尝试转换一个负责压缩文件夹及其子文件夹的程序部分时，我遇到了问题。这是我目前的成果——我会在最后列出一些帮助我的来源：

from Queue import Queue

import os
import gtk
import threading
import time
import zipfile

debug = True

class ProduceToQueue( threading.Thread ):
    def __init__( self, threadName, queue, window, maximum, zipobj, dirPath ):
        threading.Thread.__init__( self, name = threadName )
        self.sharedObject = queue
        self.maximum = maximum
        self.zip = zipobj
        self.dirPath = dirPath
        self.window = window

        global debug

        if debug:
            print self.getName(), "got all params."

    def run( self ):
        if debug:
            print "Beginning zip."
        files = 0
        parentDir, dirToZip = os.path.split(self.dirPath)
        includeDirInZip = False
        #Little nested function to prepare the proper archive path
        def trimPath(path):
            archivePath = path.replace(parentDir, "", 1)
            if parentDir:
                archivePath = archivePath.replace(os.path.sep, "", 1)
            if not includeDirInZip:
                archivePath = archivePath.replace(dirToZip + os.path.sep, "", 1)
            return os.path.normcase(archivePath)
        for (archiveDirPath, dirNames, fileNames) in os.walk(self.dirPath):
            #if debug:
                #print "Walking path..."
            for fileName in fileNames:
                time.sleep( 0.001 )
                #if debug:
                    #print "After a small sleep, I'll start zipping."
                filePath = os.path.join(archiveDirPath, fileName)
                self.zip.write(filePath, trimPath(filePath))
                #if debug:
                    #print "File zipped - ",
                files = files + 1
                #if debug:
                    #print "I now have ", files, " files in the zip."
                self.sharedObject.put( files )
            #Make sure we get empty directories as well
            if not fileNames and not dirNames:
                zipInfo = zipfile.ZipInfo(trimPath(archiveDirPath) + "/")
                #some web sites suggest doing
                #zipInfo.external_attr = 16
                #or
                #zipInfo.external_attr = 48
                #Here to allow for inserting an empty directory.  Still TBD/TODO.
                outFile.writestr(zipInfo, "")

class ConsumeFromQueue( threading.Thread ):
    def __init__( self, threadName, queue, window, maximum ):
        threading.Thread.__init__( self, name = threadName )
        self.sharedObject = queue
        self.maximum = maximum
        self.window = window

        global debug

        if debug:
            print self.getName(), "got all params."

    def run( self ):
        print "Beginning progress bar update."
        for i in range( self.maximum ):
            time.sleep( 0.001 )
            #if debug:
                #print "After a small sleep, I'll get cracking on those numbers."
            current = self.sharedObject.get()
            fraction = current / float(self.maximum)
            self.window.progress_bar.set_fraction(fraction)
            #if debug:
                #print "Progress bar updated."

class MainWindow(gtk.Window):
    def __init__(self):
        super(MainWindow, self).__init__()
        self.connect("destroy", gtk.main_quit)
        vb = gtk.VBox()
        self.add(vb)
        self.progress_bar = gtk.ProgressBar()
        vb.pack_start(self.progress_bar)
        b = gtk.Button(stock=gtk.STOCK_OK)
        vb.pack_start(b)
        b.connect('clicked', self.on_button_clicked)
        b2 = gtk.Button(stock=gtk.STOCK_CLOSE)
        vb.pack_start(b2)
        b2.connect('clicked', self.on_close_clicked)
        self.show_all()

        global debug

    def on_button_clicked(self, button):
        folder_to_zip = "/home/user/folder/with/lotsoffiles"
        file_count = sum((len(f) + len(d) for _, d, f in os.walk(folder_to_zip)))
        outFile = zipfile.ZipFile("/home/user/multithreadziptest.zip", "w", compression=zipfile.ZIP_DEFLATED)

        queue = Queue()

        producer = ProduceToQueue("Zipper", queue, self, file_count, outFile, folder_to_zip)
        consumer = ConsumeFromQueue("ProgressBar", queue, self, file_count)

        producer.start()
        consumer.start()

        producer.join()
        consumer.join()

        outFile.close()

        if debug:
            print "Done!"

    def on_close_clicked(self, widget):
        gtk.main_quit()

w = MainWindow()
gtk.main()

问题是，当文件数量达到大约7000个时，程序就会卡住，我只能强制退出。我还没有尝试过更少的文件数量，但我觉得可能也会遇到同样的问题。此外，进度条也没有更新。我知道我的代码风格很乱（混用了下划线和驼峰命名法，还有一些新手错误），但我想不出有什么理由让它不工作。

这是我大部分内容的来源：

http://www.java2s.com/Code/Python/File/Multithreadingzipfile.htm http://www.java2s.com/Tutorial/Python/0340__Thread/Multiplethreadsproducingconsumingvalues.htm

多线程文件处理代码风格进度条线程程序优化压缩文件应用程序移植

Python zipfile 多线程与进度条结合使用

1 个回答

撰写回答