How to find frame outlines from an Android camera and convert them to box2d bodies?

Question

Using OpenFrameworks, OpenCV and Box2D, I was able to achieve this with a good frame rate. Using Android seems a lot more challenging (partly because I'm new to JAVA).

Here's how I started:

• Use "OpenCV Image Samples - Images" and delete everything except the "canny" effect, which creates a beautiful black-and-white image, which is ideal for finding contours.

  public Mat onCameraFrame(CvCameraViewFrame inputFrame) { mRgba = inputFrame.rgba(); Imgproc.Canny(mRgbaInnerWindow, mIntermediateMat, 50, 100); Imgproc.cvtColor(mIntermediateMat, mRgbaInnerWindow, Imgproc.COLOR_GRAY2BGRA, 4); return mRgba; } 

• From "OpenCV Sample - Defining Color-blob", I grabbed the logic to find the contours in Mat:

   // These two lines are actually in the function onCameraViewStarted mHierarchy = new Mat(); CONTOUR_COLOR = new Scalar(255,0,0,255); // These lines are in function onCameraFrame List<MatOfPoint> contours = new ArrayList<MatOfPoint>(); Imgproc.findContours(mRgbaInnerWindow, contours, mHierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); Imgproc.drawContours(mIntermediateMat, contours, -1, CONTOUR_COLOR); 

  So, my current function looks like this and it does not work:

   public Mat onCameraFrame(CvCameraViewFrame inputFrame) { mRgba = inputFrame.rgba(); if ((mRgbaInnerWindow == null) || (mGrayInnerWindow == null) || (mRgba.cols() != mSizeRgba.width) || (mRgba.height() != mSizeRgba.height)) CreateAuxiliaryMats(); Imgproc.Canny(mRgbaInnerWindow, mIntermediateMat, 50, 100); //Imgproc.cvtColor(mIntermediateMat, mRgbaInnerWindow, Imgproc.COLOR_GRAY2BGRA, 4); List<MatOfPoint> contours = new ArrayList<MatOfPoint>(); Imgproc.findContours(mRgbaInnerWindow, contours, mHierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); //Imgproc.drawContours(mIntermediateMat, contours, -1, CONTOUR_COLOR); return mRgba; } 

• Now this is where I am stuck. I keep getting exceptions and I think that I am not using the correct measurements or turning Mat into the right color space. This post has some insight, but I don't know if this is correct: OpenCV on Android findContours throws Exception

Answer 1

Hi, I'm new to openCV too, but this code may help,

 import android.app.Activity; import android.os.Bundle; import android.util.Log; import android.view.SurfaceView; import org.opencv.android.*; import org.opencv.core.*; import org.opencv.imgproc.Imgproc; import java.util.ArrayList; import java.util.List; public class MainActivity extends Activity implements CvCameraViewListener2{ private static final String TAG = MainActivity.class.getCanonicalName(); private CameraBridgeViewBase mOpenCvCameraView; private Mat mRgba; private Mat mIntermediateMat; private Mat mGray; Mat hierarchy; List<MatOfPoint> contours; private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) { @Override public void onManagerConnected(int status) { switch (status) { case LoaderCallbackInterface.SUCCESS: { Log.i(TAG, "OpenCV loaded successfully"); mOpenCvCameraView.enableView(); } break; default: { super.onManagerConnected(status); } break; } } }; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_main); mOpenCvCameraView = (CameraBridgeViewBase) findViewById(R.id.java_surface_view); mOpenCvCameraView.setVisibility(SurfaceView.VISIBLE); mOpenCvCameraView.setCvCameraViewListener(this); } @Override public void onResume() { super.onResume(); OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_2_4_3, this, mLoaderCallback); } @Override public void onPause() { super.onPause(); if (mOpenCvCameraView != null) mOpenCvCameraView.disableView(); } @Override public void onCameraViewStarted(int width, int height) { mRgba = new Mat(height, width, CvType.CV_8UC4); mIntermediateMat = new Mat(height, width, CvType.CV_8UC4); mGray = new Mat(height, width, CvType.CV_8UC1); hierarchy = new Mat(); } @Override public void onCameraViewStopped() { mRgba.release(); mGray.release(); mIntermediateMat.release(); hierarchy.release(); } @Override public Mat onCameraFrame(CvCameraViewFrame inputFrame) { mRgba = inputFrame.gray(); contours = new ArrayList<MatOfPoint>(); hierarchy = new Mat(); Imgproc.Canny(mRgba, mIntermediateMat, 80, 100); Imgproc.findContours(mIntermediateMat, contours, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0, 0)); /* Mat drawing = Mat.zeros( mIntermediateMat.size(), CvType.CV_8UC3 ); for( int i = 0; i< contours.size(); i++ ) { Scalar color =new Scalar(Math.random()*255, Math.random()*255, Math.random()*255); Imgproc.drawContours( drawing, contours, i, color, 2, 8, hierarchy, 0, new Point() ); }*/ hierarchy.release(); Imgproc.drawContours(mRgba, contours, -1, new Scalar(Math.random()*255, Math.random()*255, Math.random()*255));//, 2, 8, hierarchy, 0, new Point()); // Imgproc.cvtColor(mIntermediateMat, mRgba, Imgproc.COLOR_GRAY2RGBA, 4); return mRgba; } } 

I know that this may not be the best way to achieve this, but we are all here to learn new ways :)

Answer 2

Short answer: you should use the Canny result as input (1st argument) for findContours , i.e.

 Imgproc.findContours(mIntermediateMat, ...);