如何改进立体对图像的视差图（python）

Question

我从手机上抓拍了以下两张照片：Image1，Image2

摄像机已校准，我使用此代码重建3D浊点：

'''
Created by Omar Padierna "Para11ax" on Jan 1 2019

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.


'''
import cv2
import numpy as np
import glob
from tqdm import tqdm
import PIL.ExifTags
import PIL.Image
from matplotlib import pyplot as plt

#=====================================
# Function declarations
#=====================================

#Function to create point cloud file
def create_output(vertices, colors, filename):
    colors = colors.reshape(-1,3)
    vertices = np.hstack([vertices.reshape(-1,3),colors])

    ply_header = '''ply
        format ascii 1.0
        element vertex %(vert_num)d
        property float x
        property float y
        property float z
        property uchar red
        property uchar green
        property uchar blue
        end_header
        '''
    with open(filename, 'w') as f:
        f.write(ply_header %dict(vert_num=len(vertices)))
        np.savetxt(f,vertices,'%f %f %f %d %d %d')

#Function that Downsamples image x number (reduce_factor) of times.
def downsample_image(image, reduce_factor):
    for i in range(0,reduce_factor):
        #Check if image is color or grayscale
        if len(image.shape) > 2:
            row,col = image.shape[:2]
        else:
            row,col = image.shape

        image = cv2.pyrDown(image, dstsize= (col//2, row // 2))
    return image


#=========================================================
# Stereo 3D reconstruction
#=========================================================

#Load camera parameters
ret = np.load('D:/Books/Pav Man/3DReconstruction-master/Reconstruction/camera_params/ret.npy')
K = np.load('D:/Books/Pav Man/3DReconstruction-master/Reconstruction/camera_params/K.npy')
dist = np.load('D:/Books/Pav Man/3DReconstruction-master/Reconstruction/camera_params/dist.npy')

#Specify image paths
img_path1 = 'D:/Books/Pav Man/3DReconstruction-master/Reconstruction/reconstruct_this/TestVi4.jpg'
img_path2 = 'D:/Books/Pav Man/3DReconstruction-master/Reconstruction/reconstruct_this/TestVi5.jpg'

#Load pictures
img_1 = cv2.imread(img_path1)
img_2 = cv2.imread(img_path2)

#Get height and width. Note: It assumes that both pictures are the same size. They HAVE to be same size and height.
h,w = img_2.shape[:2]

#Get optimal camera matrix for better undistortion
new_camera_matrix, roi = cv2.getOptimalNewCameraMatrix(K,dist,(w,h),1,(w,h))

#Undistort images
img_1_undistorted = cv2.undistort(img_1, K, dist, None, new_camera_matrix)
img_2_undistorted = cv2.undistort(img_2, K, dist, None, new_camera_matrix)

#Downsample each image 3 times (because they're too big)
img_1_downsampled = downsample_image(img_1_undistorted,3)
img_2_downsampled = downsample_image(img_2_undistorted,3)

#cv2.imwrite('undistorted_left.jpg', img_1_downsampled)
#cv2.imwrite('undistorted_right.jpg', img_2_downsampled)


#Set disparity parameters
#Note: disparity range is tuned according to specific parameters obtained through trial and error.
win_size = 1
min_disp = -1
max_disp = abs(min_disp) * 9 #min_disp * 9
num_disp = max_disp - min_disp # Needs to be divisible by 16

#Create Block matching object.
stereo = cv2.StereoSGBM_create(minDisparity= min_disp,
    numDisparities = num_disp,
    blockSize = 5,
    uniquenessRatio = 5,
    speckleWindowSize = 1,
    speckleRange = 5,
    disp12MaxDiff = 2,
    P1 = 8*3*win_size**2,#8*3*win_size**2,
    P2 =32*3*win_size**2) #32*3*win_size**2)

#Compute disparity map
print ("\nComputing the disparity  map...")
disparity_map = stereo.compute(img_1_downsampled, img_2_downsampled)

#Show disparity map before generating 3D cloud to verify that point cloud will be usable.
plt.imshow(disparity_map,'gray')
plt.show()

#Generate  point cloud.
print ("\nGenerating the 3D map...")

#Get new downsampled width and height
h,w = img_2_downsampled.shape[:2]

#Load focal length.
focal_length = np.load('D:/Books/Pav Man/3DReconstruction-master/Reconstruction/camera_params/FocalLength.npy')

#Perspective transformation matrix
#This transformation matrix is from the openCV documentation, didn't seem to work for me.
Q = np.float32([[1,0,0,-w/2.0],
                [0,-1,0,h/2.0],
                [0,0,0,-focal_length],
                [0,0,1,0]])

#This transformation matrix is derived from Prof. Didier Stricker's power point presentation on computer vision.
#Link : https://ags.cs.uni-kl.de/fileadmin/inf_ags/3dcv-ws14-15/3DCV_lec01_camera.pdf
Q2 = np.float32([[1,0,0,0],
                [0,-1,0,0],
                [0,0,focal_length*0.05,0], #Focal length multiplication obtained experimentally.
                [0,0,0,1]])

#Reproject points into 3D
points_3D = cv2.reprojectImageTo3D(disparity_map, Q2)
#Get color points
colors = cv2.cvtColor(img_1_downsampled, cv2.COLOR_BGR2RGB)

#Get rid of points with value 0 (i.e no depth)
mask_map = disparity_map > disparity_map.min()

#Mask colors and points.
output_points = points_3D[mask_map]
output_colors = colors[mask_map]

#Define name for output file
output_file = 'D:/Books/Pav Man/3DReconstruction-master/Reconstruction/reconstructed.ply'

#Generate point cloud
print ("\n Creating the output file... \n")
create_output(output_points, output_colors, output_file)

以下图像用于视差图和3D模型： Disparity map ，Model 正如您在模型图像中看到的，存在空白区域（红色区域），如何用点填充此区域，以及如何改进视差贴图

Answer 1

非置信区域（即，算法不确定这是正确的视差）在视差图中标记为黑色像素。这是意料之中的行为

你必须做一些后期处理来填充地图。使用引导过滤器完成映射。如果matlab提供任何引导过滤器，您可以尝试一次。在OpenCV中，“WLS”是获取填充贴图最常用的引导过滤器