Having trouble with Camera calibration for stereovision in open CV - Page 1
Forum section: electronics / general

author: Refrigerator
author_rank: Super Contributor
timestamp: April 03, 2024, 06:33:37 pm
content: I'm revisiting my old openCV project, where i need to get some 3d-ish imaging from two stereo cameras but i'm having trouble with calibration.The code i'm using is just example code from the internet (Nicolai Nielsen i believe), because what i need is the just end result - a stereo camera setup that produces a depth map.Code:# import libsimport numpy as npimport cv2 as cvimport globimport sys################ FIND CHESSBOARD CORNERS - OBJECT POINTS AND IMAGE POINTS ############################## define chessboardchessboardSize = (7,5)# define image frame sizeframeSize = (640,480)# termination criteriacriteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)objp = np.zeros((chessboardSize * chessboardSize, 3), np.float32)objp[:,:2] = np.mgrid[0:chessboardSize,0:chessboardSize].T.reshape(-1,2)size_of_chessboard_squares_mm = 31objp = objp * size_of_chessboard_squares_mm# Arrays to store object points and image points from all the images.objpoints = [] # 3d point in real world spaceimgpointsL = [] # 2d points in image plane.imgpointsR = [] # 2d points in image plane.num = 0corners_detected = 0imagesLeft = sorted(glob.glob('images/stereoLeft/*.png'))imagesRight = sorted(glob.glob('images/stereoRight/*.png'))kernel = np.array([[0, -1, 0],[-1, 5,-1],[0, -1, 0]])for imgLeft, imgRight in zip(imagesLeft, imagesRight):num += 1imgL = cv.imread(imgLeft)imgR = cv.imread(imgRight)grayL = cv.cvtColor(imgL, cv.COLOR_BGR2GRAY)grayR = cv.cvtColor(imgR, cv.COLOR_BGR2GRAY)#grayL = cv.filter2D(grayL, ddepth=-1, kernel=kernel)#grayR = cv.filter2D(grayR, ddepth=-1, kernel=kernel)# Find the chess board cornersretL, cornersL = cv.findChessboardCorners(grayL, chessboardSize, None)retR, cornersR = cv.findChessboardCorners(grayR, chessboardSize, None)# If found, add object points, image points (after refining them)if retL and retR == True:corners_detected += 1objpoints.append(objp)cornersL = cv.cornerSubPix(grayL, cornersL, (11,11), (-1,-1), criteria)imgpointsL.append(cornersL)cornersR = cv.cornerSubPix(grayR, cornersR, (11,11), (-1,-1), criteria)imgpointsR.append(cornersR)# Draw and display the cornerscv.drawChessboardCorners(imgL, chessboardSize, cornersL, retL)cv.imshow('img left', imgL)cv.drawChessboardCorners(imgR, chessboardSize, cornersR, retR)cv.imshow('img right', imgR)cv.waitKey(10)else:print("chessboard corners not found in image " + str(num))cv.destroyAllWindows()# terminate program is no corners are detectedif corners_detected == 0:sys.exit("No corners detected")############## CALIBRATION #######################################################retL, cameraMatrixL, distL, rvecsL, tvecsL = cv.calibrateCamera(objpoints, imgpointsL, frameSize, None, None)heightL, widthL, channelsL = imgL.shapenewCameraMatrixL, roi_L = cv.getOptimalNewCameraMatrix(cameraMatrixL, distL, (widthL, heightL), 1, (widthL, heightL))retR, cameraMatrixR, distR, rvecsR, tvecsR = cv.calibrateCamera(objpoints, imgpointsR, frameSize, None, None)heightR, widthR, channelsR = imgR.shapenewCameraMatrixR, roi_R = cv.getOptimalNewCameraMatrix(cameraMatrixR, distR, (widthR, heightR), 1, (widthR, heightR))# assign ports to camscamR = cv.VideoCapture(0, cv.CAP_DSHOW)camL = cv.VideoCapture(1, cv.CAP_DSHOW)num = 0avg_img = 0# morphology settingskernel = np.ones((10,10),np.uint8)# begin loopwhile camL.isOpened():#frame_counter += 1# capture image frameresultL, imgL = camL.read()resultR, imgR = camR.read()Right_grey = cv.cvtColor(imgR, cv.COLOR_BGR2GRAY)Left_grey = cv.cvtColor(imgL, cv.COLOR_BGR2GRAY)# undistort image# undistortRight_grey = cv.undistort(Right_grey, cameraMatrixR, distR, None, newCameraMatrixR)Left_grey = cv.undistort(Left_grey, cameraMatrixL, distL, None, newCameraMatrixL)# resize imagesxR, yR, wR, hR = roi_RxL, yL, wL, hL = roi_L# Right_grey = Right_grey[yR:yR + hR, xR:xR + wR]# Left_grey = Left_grey[yL:yL + hL, xL:xL + wL]num_disp = 16*8size_block = 15test_no = 3#stereo = cv.StereoSGBM_create(numDisparities=num_disp, blockSize=size_block)stereo = cv.StereoBM_create(numDisparities=num_disp, blockSize=size_block)disparity = stereo.compute(Left_grey,Right_grey)#plt.imshow(disparity,'gray')#plt.show()# apply morphological transformation#disparity = cv.morphologyEx(disparity, cv.MORPH_OPEN, kernel)norm_image = cv.normalize(disparity, None, alpha = 0, beta = 1, norm_type=cv.NORM_MINMAX, dtype=cv.CV_32F)#norm_image = cv.medianBlur(norm_image,3)# wait for key pressk = cv.waitKey(5)file_ident = "_nD" + str(num_disp) + "_sB" + str(size_block) + "_test" + str(test_no) + "_Run"if k == 27:breakelif k == ord('s'): # wait for 's' key to save and exitcv.imwrite('disparity test without cal/test recal2/Uncorrected_' + file_ident + str(num) + '.png', imgR)cv.imwrite('disparity test without cal/test recal2/corrected_' + file_ident + str(num) + '.png', cv.undistort(imgR, cameraMatrixR, distR, None, newCameraMatrixR))cv.imwrite('disparity test without cal/test recal2/disparity_' + file_ident + str(num) + '.png', disparity)print("images saved")num += 1avg_img += norm_imageavg_img = avg_img / 2cv.imshow('filtered image disparity', avg_img)cv.imshow('normalized image disparity', norm_image)cv.imshow('Test Window Left',imgL)cv.imshow('Test Window Right',imgR)cv.imshow('undistorted right', cv.undistort(imgR, cameraMatrixR, distR, None, newCameraMatrixR))# show regular disparity#cv.imshow('regular disparity', imgR - imgL)cv.imshow('regular disparity mono', Right_grey - Left_grey)# Release and destroy all windows before terminationcamL.release()camR.release()cv.destroyAllWindows()Unfortunately i'm having trouble calibrating the distortion and alignment of my two cameras.My problem is that the end result is never perfectly aligned or calibrated and i can't find the reason why, because i followed tutorials to a T and sometimes my calibrated image comes out worse than before.I'm trying to troubleshoot my system because i have the classic deadline-coming-up crunch session and not enough time.The cameras i'm using are 1080p native web cameras, that stream 480p images, because that's what my code is set to (to ease the processing a bit). These cameras seem to have pretty good sensors.I've included some images to show what i'm working with. First is the calibration image with checked board, which seems to detect just fine. Second is the absolute best i could after multiple days of non stop recalibrating and in the end having to tweak the cameras by hand to align the images.The images may look blurry, because they're just screenshots but in reality they're quite high fidelity and all checkerboard corners identify just fine, which is why i don't understand how my final image becomes so distorted.Perhaps there's not enough light? Maybe there's a problem with reading 480p images from a 1080p webcam? Are the lenses just too wide-angle for this code to work? My cameras were about 20cm apart, mounted on a 20x40mm aluminum extrusion with 3D printed holders, for solid mounting.I mean there could be some simple detail i'm missing out on, like the autofocus in my cameras messing something up.Any help would be greatly appreciated.