Geek Answers Handbook

Planning for Android Camera.PreviewCallback (with OpenGL and OpenCV)

I am developing an application that requires intensive image processing using camera input and displaying results in real time. I decided to use OpenGL and OpenCV together with the usual Android camera API. Until now, it has become a bit of a multi-threaded nightmare, and unfortunately I feel very limited due to the lack of documentation in the onPreviewFrame () callback.

I know from the documentation that onPreviewFrame () is called on a thread that acquires a camera using Camera.open (). What confuses me is how this callback is planned - it seems to have a fixed frame rate. My current architecture relies on the onPreviewFrame () callback to initiate the image processing / display loop, and it seems to get stuck when I block the camera callback thread for too long, so I suspect the callback is inflexible when it comes to comes to planning, I would like to slow down the frame rate to check this out, but my device does not support this.

I started with the code at http://maninara.blogspot.ca/2012/09/render-camera-preview-using-opengl-es.html . This code is not very parallel, and it is only intended to display the data that the camera returns. For my needs, I adapted the code for drawing raster images, and I use a dedicated stream to buffer camera data into another dedicated heavy image processing stream (all outside of the OpenGL stream).

Here is my code (simplified):

CameraSurfaceRenderer.java

class CameraSurfaceRenderer implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener,
    Camera.PreviewCallback
{

static int[]                surfaceTexPtr;

static CameraSurfaceView    cameraSurfaceView;
static FloatBuffer          pVertex;
static FloatBuffer          pTexCoord;
static int                  hProgramPointer;

static Camera               camera;
static SurfaceTexture       surfaceTexture;

static Bitmap               procBitmap;
static int[]                procBitmapPtr;

static boolean              updateSurfaceTex = false;

static ConditionVariable    previewFrameLock;
static ConditionVariable    bitmapDrawLock;

// MarkerFinder extends CameraImgProc
static MarkerFinder         markerFinder = new MarkerFinder();
static Thread               previewCallbackThread;

static
{
    previewFrameLock = new ConditionVariable();
    previewFrameLock.open();

    bitmapDrawLock = new ConditionVariable();
    bitmapDrawLock.open();
}

CameraSurfaceRenderer(Context context, CameraSurfaceView view)
{
    rendererContext = context;
    cameraSurfaceView = view;

    // … // Load pVertex and pTexCoord vertex buffers
}

public void close()
{
    // … // This code usually doesn’t have the chance to get called
}

@Override
public void onSurfaceCreated(GL10 unused, EGLConfig config)
{
// .. // Initialize a texture object for the bitmap data

    surfaceTexPtr = new int[1];
    surfaceTexture = new SurfaceTexture(surfaceTexPtr[0]);
    surfaceTexture.setOnFrameAvailableListener(this);

    //Initialize camera on its own thread so preview frame callbacks are processed in parallel
    previewCallbackThread = new Thread()
    {
        @Override
        public void run()
        {
            try {
                camera = Camera.open();
            } catch (RuntimeException e) {
                // … // Bitch to the user through a Toast on the UI thread
            }
            assert camera != null;
            //Callback set on CameraSurfaceRenderer class, but executed on worker thread
            camera.setPreviewCallback(CameraSurfaceRenderer.this);
            try {
                camera.setPreviewTexture(surfaceTexture);
            } catch (IOException e) {
                Log.e(Const.TAG, "Unable to set preview texture");
            }

            Looper.prepare();
            Looper.loop();
        }
    };
    previewCallbackThread.start();

   // … // More OpenGL initialization stuff
}

@Override
public void onDrawFrame(GL10 unused)
{
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);

    synchronized (this)
    {
        surfaceTexture.updateTexImage();
    }

// Binds bitmap data to texture
    bindBitmap(procBitmap);

// … // Acquire shader program ttributes, render
    GLES20.glFlush();
}

@Override
public synchronized void onFrameAvailable(SurfaceTexture surfaceTexture)
{
    cameraSurfaceView.requestRender();
}

@Override
public void onPreviewFrame(byte[] data, Camera camera)
{
    Bitmap bitmap = markerFinder.exchangeRawDataForProcessedImg(data, null, camera);

    // … // Check for null bitmap

    previewFrameLock.block();

    procBitmap = bitmap;

    previewFrameLock.close();
    bitmapDrawLock.open();
}

void bindBitmap(Bitmap bitmap)
{
    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, procBitmapPtr[0]);

    bitmapDrawLock.block();

    if (bitmap != null && !bitmap.isRecycled())
    {
        GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0);
        bitmap.recycle();
    }

    bitmapDrawLock.close();
    previewFrameLock.open();
}

@Override
public void onSurfaceChanged(GL10 unused, int width, int height)
{
    GLES20.glViewport(0, 0, width, height);

    // … // Set camera parameters

    camera.startPreview();
}

void deleteTexture()
{
    GLES20.glDeleteTextures(1, surfaceTexPtr, 0);
}
}

CameraImgProc.java (abstract class)

public abstract class CameraImgProc
{
CameraImgProcThread  thread = new CameraImgProcThread();
Handler              handler;
ConditionVariable    bufferSwapLock = new ConditionVariable(true);
Runnable             processTask = new Runnable()
{
    @Override
    public void run()
    {
        imgProcBitmap = processImg(lastWidth, lastHeight, cameraDataBuffer, imgProcBitmap);
        bufferSwapLock.open();
    }
};

int lastWidth    = 0;
int lastHeight   = 0;

Mat cameraDataBuffer;
Bitmap imgProcBitmap;

public CameraImgProc()
{
    thread.start();
    handler = thread.getHandler();
}

protected abstract Bitmap allocateBitmapBuffer(int width, int height);

public final Bitmap exchangeRawDataForProcessedImg(byte[] data, Bitmap dirtyBuffer, Camera camera)
{
    Camera.Parameters parameters = camera.getParameters();
    Camera.Size size = parameters.getPreviewSize();

    // Wait for worker thread to finish processing image
    bufferSwapLock.block();
    bufferSwapLock.close();

    Bitmap freshBuffer = imgProcBitmap;
    imgProcBitmap = dirtyBuffer;

    // Reallocate buffers if size changes to avoid overflow
    assert size != null;
    if (lastWidth != size.width || lastHeight != size.height)
    {
        lastHeight  = size.height;
        lastWidth   = size.width;

        if (cameraDataBuffer != null) cameraDataBuffer.release();
        //YUV format requires 1.5 times as much information in vertical direction
        cameraDataBuffer = new Mat((lastHeight * 3) / 2, lastWidth, CvType.CV_8UC1);

        imgProcBitmap = allocateBitmapBuffer(lastWidth, lastHeight);
        // Buffers had to be resized, therefore no processed data to return

        cameraDataBuffer.put(0, 0, data);

        handler.post(processTask);
        return null;
    }

    // If program did not pass a buffer
    if (imgProcBitmap == null)
        imgProcBitmap = allocateBitmapBuffer(lastWidth, lastHeight);

    // Exchange data
    cameraDataBuffer.put(0, 0, data);

    // Give img processing task to worker thread
    handler.post(processTask);

    return freshBuffer;
}

protected abstract Bitmap processImg(int width, int height, Mat cameraData, Bitmap dirtyBuffer);

class CameraImgProcThread extends Thread
{
    volatile Handler handler;

    @Override
    public void run()
    {
        Looper.prepare();
        handler = new Handler();
        Looper.loop();
    }

    Handler getHandler()
    {
        //noinspection StatementWithEmptyBody
        while (handler == null)
        {
            try {
                Thread.currentThread();
                Thread.sleep(5);
            } catch (Exception e) {
                //Do nothing
            }
        };
        return handler;
    }
}
}

, , , CameraImgProc.processImg(). , , , , , , .

:

Camera.PreviewCallback ?

Android API ?

, ?

+4
2

onPreviewFrame() , Camera.open()

. , , "" . , UI, " ", HandlerThread. , SO. , , , , , , .

: , .

setOneShotPreviewCallback(), 1 FPS . , , setPreviewCallbackWithBuffer return onPreviewFrame(), . , , .

, , : 200 , . [] . , , , , undefined. , , , , .

+7

( > 4.0) . , . ; , HAL, . ( onPreviewFrame), , . , .

+1

More articles:


All Articles