Calculation of aspect ratio of perspective image

Question

I recently implemented Prospective Conversion to OpenCV for my Android application. Almost everything works without problems, but one aspect requires much more work.

The problem is that I don’t know how to calculate the correct aspect ratio of the target Perspective Transform image (it does not need to be set manually) so that it can calculate the aspect ratio of the image to the size of the real thing / image , despite the camera angle . Please note that the initial coordinates do not form a trapezoid, it forms a quadrangle.

If I have a photograph of a book taken from approximately 45 degrees, and I want the aspect ratio of the image to be approximately the same as the aspect ratio of this book. It's hard to take a 2D photo, but CamScanner does it perfectly. I made a very simple way to calculate the size of my target image (without any expectation that it works the way I want), but it makes the image with the angle of 45 degrees about 20% shorter and when the angle decreases, the image height decreases significantly, in while CamScanner does it perfectly, despite the angle:

Here CamScanner supports the aspect ratio of the target image (second), the same as in the book, it is pretty accurate even at ~ 20 degrees.

Meanwhile, my code looks like this (when calculating the size of the target image, I have no intention of working, as I ask in this question):

public static Mat PerspectiveTransform(Point[] cropCoordinates, float ratioW, float ratioH, Bitmap croppedImage) { if (cropCoordinates.length != 4) return null; double width1, width2, height1, height2, avgw, avgh; Mat src = new Mat(); List<Point> startCoords = new ArrayList<>(); List<Point> resultCoords = new ArrayList<>(); Utils.bitmapToMat(croppedImage, src); for (int i = 0; i < 4; i++) { if (cropCoordinates[i].y < 0 ) new Point(cropCoordinates[i].x, 0); startCoords.add(new Point(cropCoordinates[i].x * ratioW, cropCoordinates[i].y * ratioH)); } width1 = Math.sqrt(Math.pow(startCoords.get(2).x - startCoords.get(3).x,2) + Math.pow(startCoords.get(2).y - startCoords.get(3).y,2)); width2 = Math.sqrt(Math.pow(startCoords.get(1).x - startCoords.get(0).x,2) + Math.pow(startCoords.get(1).y - startCoords.get(0).y,2)); height1 = Math.sqrt(Math.pow(startCoords.get(1).x - startCoords.get(2).x, 2) + Math.pow(startCoords.get(1).y - startCoords.get(2).y, 2)); height2 = Math.sqrt(Math.pow(startCoords.get(0).x - startCoords.get(3).x, 2) + Math.pow(startCoords.get(0).y - startCoords.get(3).y, 2)); avgw = (width1 + width2) / 2; avgh = (height1 + height2) / 2; resultCoords.add(new Point(0, 0)); resultCoords.add(new Point(avgw-1, 0)); resultCoords.add(new Point(avgw-1, avgh-1)); resultCoords.add(new Point(0, avgh-1)); Mat start = Converters.vector_Point2f_to_Mat(startCoords); Mat result = Converters.vector_Point2d_to_Mat(resultCoords); start.convertTo(start, CvType.CV_32FC2); result.convertTo(result,CvType.CV_32FC2); Mat mat = new Mat(); Mat perspective = Imgproc.getPerspectiveTransform(start, result); Imgproc.warpPerspective(src, mat, perspective, new Size(avgw, avgh)); return mat; } 

And from relatively the same angle, my method produces this result:

I want to know how to do this? I wonder how they managed to calculate the length of the object, just having the coordinates of the four corners. Also, if possible, provide some code / math explanations or articles of a similar / same.

Thanks in advance.

Answer 1

This happened several times before on SO, but I never saw the full answer, so here. The implementation presented here is based on this article, which derives the complete equations: http://research.microsoft.com/en-us/um/people/zhang/papers/tr03-39.pdf

Essentially, this shows that assuming a pinhole camera model, you can calculate the aspect ratio for the projected rectangle (but not the scale, not surprisingly). Essentially, you can decide the focal length, then get the aspect ratio. Here's an example implementation in python using OpenCV. Note that you need to have 4 detected angles in the correct order or not work (pay attention to the order, this is a zigzag). Reported error rates are in the range of 3-5%.

 import math import cv2 import scipy.spatial.distance import numpy as np img = cv2.imread('img.png') (rows,cols,_) = img.shape #image center u0 = (cols)/2.0 v0 = (rows)/2.0 #detected corners on the original image p = [] p.append((67,74)) p.append((270,64)) p.append((10,344)) p.append((343,331)) #widths and heights of the projected image w1 = scipy.spatial.distance.euclidean(p[0],p[1]) w2 = scipy.spatial.distance.euclidean(p[2],p[3]) h1 = scipy.spatial.distance.euclidean(p[0],p[2]) h2 = scipy.spatial.distance.euclidean(p[1],p[3]) w = max(w1,w2) h = max(h1,h2) #visible aspect ratio ar_vis = float(w)/float(h) #make numpy arrays and append 1 for linear algebra m1 = np.array((p[0][0],p[0][1],1)).astype('float32') m2 = np.array((p[1][0],p[1][1],1)).astype('float32') m3 = np.array((p[2][0],p[2][1],1)).astype('float32') m4 = np.array((p[3][0],p[3][1],1)).astype('float32') #calculate the focal disrance k2 = np.dot(np.cross(m1,m4),m3) / np.dot(np.cross(m2,m4),m3) k3 = np.dot(np.cross(m1,m4),m2) / np.dot(np.cross(m3,m4),m2) n2 = k2 * m2 - m1 n3 = k3 * m3 - m1 n21 = n2[0] n22 = n2[1] n23 = n2[2] n31 = n3[0] n32 = n3[1] n33 = n3[2] f = math.sqrt(- (1.0/(n23*n33)) * ((n21*n31 - (n21*n33 + n23*n31)*u0 + n23*n33*u0*u0) + (n22*n32 - (n22*n33+n23*n32)*v0 + n23*n33*v0*v0))) A = np.array([[f,0,u0],[0,f,v0],[0,0,1]]).astype('float32') At = np.transpose(A) Ati = np.linalg.inv(At) Ai = np.linalg.inv(A) #calculate the real aspect ratio ar_real = math.sqrt(np.dot(np.dot(np.dot(n2,Ati),Ai),n2)/np.dot(np.dot(np.dot(n3,Ati),Ai),n3)) if ar_real < ar_vis: W = int(w) H = int(W / ar_real) else: H = int(h) W = int(ar_real * H) pts1 = np.array(p).astype('float32') pts2 = np.float32([[0,0],[W,0],[0,H],[W,H]]) #project the image with the new w/h M = cv2.getPerspectiveTransform(pts1,pts2) dst = cv2.warpPerspective(img,M,(W,H)) cv2.imshow('img',img) cv2.imshow('dst',dst) cv2.imwrite('orig.png',img) cv2.imwrite('proj.png',dst) cv2.waitKey(0) 

Original:

Projected (resolution is very low, since I cropped the image from the screen, but the aspect ratio seems to be correct):

Answer 2

Thanks y300 and this post fooobar.com/questions/137846 / ... I got it in Java. I will leave it here if someone has the same problems that I converted it to Java ...

 public float getRealAspectRatio(int imageWidth, int imageHeight) { double u0 = imageWidth/2; double v0 = imageHeight/2; double m1x = mTopLeft.x - u0; double m1y = mTopLeft.y - v0; double m2x = mTopRight.x - u0; double m2y = mTopRight.y - v0; double m3x = mBottomLeft.x - u0; double m3y = mBottomLeft.y - v0; double m4x = mBottomRight.x - u0; double m4y = mBottomRight.y - v0; double k2 = ((m1y - m4y)*m3x - (m1x - m4x)*m3y + m1x*m4y - m1y*m4x) / ((m2y - m4y)*m3x - (m2x - m4x)*m3y + m2x*m4y - m2y*m4x) ; double k3 = ((m1y - m4y)*m2x - (m1x - m4x)*m2y + m1x*m4y - m1y*m4x) / ((m3y - m4y)*m2x - (m3x - m4x)*m2y + m3x*m4y - m3y*m4x) ; double f_squared = -((k3*m3y - m1y)*(k2*m2y - m1y) + (k3*m3x - m1x)*(k2*m2x - m1x)) / ((k3 - 1)*(k2 - 1)) ; double whRatio = Math.sqrt( (Math.pow((k2 - 1),2) + Math.pow((k2*m2y - m1y),2)/f_squared + Math.pow((k2*m2x - m1x),2)/f_squared) / (Math.pow((k3 - 1),2) + Math.pow((k3*m3y - m1y),2)/f_squared + Math.pow((k3*m3x - m1x),2)/f_squared) ) ; if (k2==1 && k3==1 ) { whRatio = Math.sqrt( (Math.pow((m2y-m1y),2) + Math.pow((m2x-m1x),2)) / (Math.pow((m3y-m1y),2) + Math.pow((m3x-m1x),2))); } return (float)(whRatio); }