在Python中并行化for循环以加速算法

Question

此算法包括读取以结尾的文件夹中的所有图像剪下的.tiff而改变所有扫描图像的伽马值的for循环将被加速。你知道吗

如何能够并行化和加速这个简单的算法？你知道吗

SinglestreetsFolder = "/dop_shapefile/"
shp_list = [x for x in os.listdir(SinglestreetsFolder) if x.endswith("clipped.tiff")]


for i in range(0, len(shp_list), 1):

    originalPath = "/dop_shapefile/" + shp_list[i]
    original = cv2.imread(originalPath)

    adjusted = adjust_gamma(original, gamma=0.3)

    cv2.imwrite("/gamma/" + shp_list[i] + "_gamma.tiff", adjusted)
    print "status_Gamma: ", i + 1, "/", len(shp_list)

---

另一种算法

此算法包括读取以.tiff结尾的文件夹中的所有图像和for循环，for循环执行ConnectedComponentLabeling算法（带有自定义项）以加快速度。你知道吗

SinglestreetsFolder = "/gabor/"
shp_list = [x for x in os.listdir(SinglestreetsFolder) if x.endswith(".tiff")]


img = cv2.imread(SinglestreetsFolder + shp_list[0], 0)
wholeimage = np.zeros(shape=(len(img), len(img[0]), 3))
erosion = np.ones((2, 2), np.uint8)     # kernel: erosion

for j in range(0, len(shp_list), 1):
    if "motorway" in shp_list[j]:
        N = 40  # pixel threshold (city: ca. 10, motorway: ca. 40)
    else:
        N = 10   # pixel threshold (city: ca. 10, motorway: ca. 40)

    img = cv2.imread(SinglestreetsFolder + shp_list[j], 0)
    connectivity = 8    # 4- OR 8-connectivity connected component labeling


    if "reverse" in shp_list[j]:
        img = cv2.threshold(img, 140, 255, cv2.THRESH_BINARY)[1]  # ensure binary

        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, erosion)

        retval, labels = cv2.connectedComponents(img, connectivity)

        num = labels.max()

        # If the count of pixels less than a threshold, then set pixels to `0` (background)
        for i in range(1, num + 1):
            pts = np.where(labels == i)
            if len(pts[0]) < N:
                labels[pts] = 0

        # Map component labels to hue val
        label_hue = np.uint8(179 * labels / np.max(labels))
        blank_ch = 255 * np.ones_like(labels)
        labeled_img = cv2.merge([blank_ch, blank_ch, blank_ch])

        # set bg label to black
        labeled_img[label_hue == 0] = 0

        wholeimage = np.where(labeled_img == 0, wholeimage, labeled_img)

    else:
        img = cv2.threshold(img, 140, 255, cv2.THRESH_BINARY)[1]  # ensure binary

        retval, labels = cv2.connectedComponents(img, connectivity)

        num = labels.max()

        # If the count of pixels less than a threshold, then set pixels to `0` (background)
        for i in range(1, num + 1):
            pts = np.where(labels == i)
            if len(pts[0]) < N:
                labels[pts] = 0

        # Map component labels to hue val
        label_hue = np.uint8(179 * labels / np.max(labels))
        blank_ch = 255 * np.ones_like(labels)
        labeled_img = cv2.merge([blank_ch, blank_ch, blank_ch])

        # set bg label to black
        labeled_img[label_hue == 0] = 0

        wholeimage = np.where(labeled_img == 0, wholeimage, labeled_img)


    print "status_CCL: ", j + 1, "/", len(shp_list)

cv2.imwrite("/ccl/streets_gabor_ccl.tiff", wholeimage)

---

我的解决方案，但“整个图像”结尾是黑色的

SinglestreetsFolder = "/gabor/"
shp_list = [x for x in os.listdir(SinglestreetsFolder) if x.endswith(".tiff")]


def gabor(params):
    img = cv2.imread(SinglestreetsFolder + shp_list[0], 0)
    wholeimage = np.zeros(shape=(len(img), len(img[0]), 3))
    erosion = np.ones((2, 2), np.uint8)  # kernel: erosion

    j, image_name = params

    if "motorway" in image_name:
        N = 40  # pixel threshold (city: ca. 10, motorway: ca. 40)
    else:
        N = 10   # pixel threshold (city: ca. 10, motorway: ca. 40)

    img = cv2.imread(SinglestreetsFolder + image_name, 0)
    connectivity = 8    # 4- OR 8-connectivity connected component labeling


    if "reverse" in image_name:
        img = cv2.threshold(img, 140, 255, cv2.THRESH_BINARY)[1]  # ensure binary

        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, erosion)
        # img = cv2.erode(img, erosion, iterations=1)

        retval, labels = cv2.connectedComponents(img, connectivity)

        num = labels.max()

        # If the count of pixels less than a threshold, then set pixels to `0` (background)
        for i in range(1, num + 1):
            pts = np.where(labels == i)
            if len(pts[0]) < N:
                labels[pts] = 0

        # Map component labels to hue val
        label_hue = np.uint8(179 * labels / np.max(labels))
        blank_ch = 255 * np.ones_like(labels)
        labeled_img = cv2.merge([blank_ch, blank_ch, blank_ch])

        # set bg label to black
        labeled_img[label_hue == 0] = 0

        wholeimage = np.where(labeled_img == 0, wholeimage, labeled_img)

    else:
        img = cv2.threshold(img, 140, 255, cv2.THRESH_BINARY)[1]  # ensure binary

        retval, labels = cv2.connectedComponents(img, connectivity)

        num = labels.max()

        # If the count of pixels less than a threshold, then set pixels to `0` (background)
        for i in range(1, num + 1):
            pts = np.where(labels == i)
            if len(pts[0]) < N:
                labels[pts] = 0

        # Map component labels to hue val
        label_hue = np.uint8(179 * labels / np.max(labels))
        blank_ch = 255 * np.ones_like(labels)
        labeled_img = cv2.merge([blank_ch, blank_ch, blank_ch])

        # set bg label to black
        labeled_img[label_hue == 0] = 0

        wholeimage = np.where(labeled_img == 0, wholeimage, labeled_img)


if __name__ == '__main__':
    p = Pool()

    list(p.imap(gabor, enumerate(shp_list)))
    cv2.imwrite(/ccl/streets_gabor_ccl.tiff", wholeimage)

Answer 1

您可以使用^{}-lib：

import threading

SinglestreetsFolder = "/dop_shapefile/"
shp_list = [x for x in os.listdir(SinglestreetsFolder) if x.endswith("clipped.tiff")]

def wrapper(i):
    originalPath = "/dop_shapefile/" + shp_list[i]
    original = cv2.imread(originalPath)
    adjusted = adjust_gamma(original, gamma=0.3)
    cv2.imwrite("/gamma/" + shp_list[i] + "_gamma.tiff", adjusted)
    print "status_Gamma: ", i + 1, "/", len(shp_list)

threads = []

for i in range(0, len(shp_list), 1):
    t = threading.Thread(target=wrapper, args=(i,))
    threads.append(t)
    t.start()

for t in threads: 
    t.join() # wait for all of them to finsih
# continue with your code

Answer 2

从假设adjust_gamma不fork子进程或使用C线程开始，图像处理是CPU限制的进程，所以在Python中应该依赖进程而不是线程。你知道吗

下面是它的外观：

from multiprocessing.pool import Pool

SinglestreetsFolder = "/dop_shapefile/"
shp_list = [x for x in os.listdir(SinglestreetsFolder) if x.endswith("clipped.tiff")]

def process_image(params):
    i, image_name = params
    originalPath = "/dop_shapefile/" + image_name
    original = cv2.imread(originalPath)

    adjusted = adjust_gamma(original, gamma=0.3)

    cv2.imwrite("/gamma/" + image_name + "_gamma.tiff", adjusted)
    return "status_Gamma: ", i + 1, "/", len(shp_list)

if __name__ == '__main__':
    p = Pool()
    results = list(p.imap(process_image, enumerate(shp_list)))
    print results

有些部分是可以改变的，我只是去调整你的例子，尽可能少的改变。你知道吗