C ++ / OpenCV - Kalman filter for video stabilization

Question

I am trying to stabilize a video with a Kalman filter for smoothing. But I have some problems.

Each time, I have two frames: one current and the other. Here is my workflow:

• Compute goodFeaturesToTrack ()
• Calculate optical flow using calcOpticalFlowPyrLK ()
• Keep only the good points.
• Appreciate Hard Transformation
• Smoothing using a Kalman filter
• Image warping.

But I think that something is wrong with Kalman, because in the end my video has not yet stabilized and it is not at all smooth, it is even worse than the original ...

Here is my Kalman code

void StabilizationTestSimple2::init_kalman(double x, double y)
{

    KF.statePre.at<float>(0) = x;
    KF.statePre.at<float>(1) = y;
    KF.statePre.at<float>(2) = 0;
    KF.statePre.at<float>(3) = 0;

    KF.transitionMatrix = *(Mat_<float>(4,4) << 1,0,1,0,    0,1,0,1,     0,0,1,0,   0,0,0,1);
    KF.processNoiseCov = *(Mat_<float>(4,4) << 0.2,0,0.2,0,  0,0.2,0,0.2,  0,0,0.3,0,
                           0,0,0,0.3);
    setIdentity(KF.measurementMatrix);
    setIdentity(KF.processNoiseCov,Scalar::all(1e-6));
    setIdentity(KF.measurementNoiseCov,Scalar::all(1e-1));
    setIdentity(KF.errorCovPost, Scalar::all(.1));
}

. . flor. cornerPrev2 cornerCurr2 , ( calcOpticalFlowPyrLK())

    /// Transformation
    Mat transformMatrix = estimateRigidTransform(cornersPrev2,cornersCurr2 ,false);

    // in rare cases no transform is found. We'll just use the last known good transform.
    if(transformMatrix.data == NULL) {
        last_transformationmatrix.copyTo(transformMatrix);
    }

    transformMatrix.copyTo(last_transformationmatrix);

    // decompose T
    double dx = transformMatrix.at<double>(0,2);
    double dy = transformMatrix.at<double>(1,2);
    double da = atan2(transformMatrix.at<double>(1,0), transformMatrix.at<double>(0,0));

    // Accumulated frame to frame transform
    x += dx;
    y += dy;
    a += da;
    std::cout << "accumulated x,y: (" << x << "," << y << ")" << endl;

    if (compteur==0){
        init_kalman(x,y);
    }
    else {


          vector<Point2f> smooth_feature_point;
          Point2f smooth_feature=kalman_predict_correct(x,y);
          smooth_feature_point.push_back(smooth_feature);
          std::cout << "smooth x,y: (" << smooth_feature.x << "," << smooth_feature.y << ")" << endl;

          // target - current
          double diff_x = smooth_feature.x - x;//
          double diff_y = smooth_feature.y - y;

          dx = dx + diff_x;
          dy = dy + diff_y;

          transformMatrix.at<double>(0,0) = cos(da);
          transformMatrix.at<double>(0,1) = -sin(da);
          transformMatrix.at<double>(1,0) = sin(da);
          transformMatrix.at<double>(1,1) = cos(da);
          transformMatrix.at<double>(0,2) = dx;
          transformMatrix.at<double>(1,2) = dy;

          warpAffine(currFrame,outImg,transformMatrix,prevFrame.size(), INTER_NEAREST|WARP_INVERSE_MAP, BORDER_CONSTANT);

          namedWindow("stabilized");
          imshow("stabilized",outImg);
          namedWindow("Original");
          imshow("Original",originalFrame);


    }

- , ?

Answer 1

KF.transitionMatrix = *(Mat_<float>(4,4) << 1,0,1,0,    0,1,0,1,     0,0,1,0,   0,0,0,1);
KF.processNoiseCov = *(Mat_<float>(4,4) << 0.2,0,0.2,0,  0,0.2,0,0.2,  0,0,0.3,0,
                       0,0,0,0.3);
setIdentity(KF.measurementMatrix);
setIdentity(KF.processNoiseCov,Scalar::all(1e-6));
setIdentity(KF.measurementNoiseCov,Scalar::all(1e-1));
setIdentity(KF.errorCovPost, Scalar::all(.1));

processNoiseCov , , . , KF .

, setIdentity , , , . , , ?

(? ?), , processNoiseCov measurementNoiseCov.

EDIT:

, , [ x_pixels, y_pixels, dx_pixels, dy_pixels ] :

• I, , , ( : , , ).
• , I, processNoiseCov measurementNoiseCov , . processNoiseCov , ( ) , . , 68% (. ), 100 , 100 * 100 = 10000 (, stddev). . (: () , , , . ).
• processNoiseCov , , 1/25 .
• measurementNoiseCov ( , - ), , .
• ( excel python Matlab - ).
• errorCovPost - , , , , .
• - KF. , , - KF.