Align already captured rgb images and depths

Question

I am trying to snap two images - one rgb and the other depth using MATLAB. Please note that I checked out a few places to do this - for example here , here , for which you want to kinect device, and here is where they say the camera settings are required for calibration. I was also suggested to use EPIPOLAR GEOMETRY to match two images, although I do not know how to do this. The dataset I am referring to is given in the rgb-dt face dataset . One such example is shown below:

, , , , . Matlab:

I = imread('1.jpg');
I1 = imcrop(I,[218,198,158,122]);
I2 = imcrop(I,[243,209,140,108]);
figure, subplot(1,2,1),imshow(I1);
subplot(1,2,2),imshow(I2);

rgb :

---

, / . , sobel rgb, , . .

.

, , .

- Matlab ?

Answer 1

:

• Kinect Kinect

, ( . . , ( 8bit - RGB), , , :

, . , :

• FOV

  , , . , . , :

  

• , FOV. , 8bit , . , . FOV . - ( ++):

  picture rgb,zed; // your input images
  struct pnt3d { float pos[3]; DWORD rgb; pnt3d(){}; pnt3d(pnt3d& a){ *this=a; }; ~pnt3d(){}; pnt3d* operator = (const pnt3d *a) { *this=*a; return this; }; /*pnt3d* operator = (const pnt3d &a) { ...copy... return this; };*/ };
  pnt3d **xyz=NULL; int xs,ys,ofsx=0,ofsy=0;
  
  void copy_images()
      {
      int x,y,x0,y0;
      float xx,yy;
      pnt3d *p;
      for (y=0;y<ys;y++)
       for (x=0;x<xs;x++)
          {
          p=&xyz[y][x];
          // copy point from depth image
          p->pos[0]=2.000*((float(x)/float(xs))-0.5);
          p->pos[1]=2.000*((float(y)/float(ys))-0.5)*(float(ys)/float(xs));
          p->pos[2]=10.0*float(DWORD(zed.p[y][x].db[0]))/255.0;
          // convert dept image x,y to color image space (FOV correction)
          xx=float(x)-(0.5*float(xs));
          yy=float(y)-(0.5*float(ys));
          xx*=98.0/108.0;
          yy*=106.0/119.0;
          xx+=0.5*float(rgb.xs);
          yy+=0.5*float(rgb.ys);
          x0=xx; x0+=ofsx;
          y0=yy; y0+=ofsy;
          // copy color from rgb image if in range
          p->rgb=0x00000000; // black
          if ((x0>=0)&&(x0<rgb.xs))
           if ((y0>=0)&&(y0<rgb.ys))
            p->rgb=rgb2bgr(rgb.p[y0][x0].dd); // OpenGL has reverse RGBorder then my image
          }
      }
  

  **xyz - 2D-, . picture - DIP, :

  • xs,ys -
  • p[ys][xs] - DWORD dd; BYTE db[4];, 32- .
  • rgb2bgr(DWORD col) RGB BGR.

• OpenGL , :

      glBegin(GL_QUADS);
      for (int y0=0,y1=1;y1<ys;y0++,y1++)
      for (int x0=0,x1=1;x1<xs;x0++,x1++)
          {
          float z,z0,z1;
          z=xyz[y0][x0].pos[2]; z0=z; z1=z0;
          z=xyz[y0][x1].pos[2]; if (z0>z) z0=z; if (z1<z) z1=z;
          z=xyz[y1][x0].pos[2]; if (z0>z) z0=z; if (z1<z) z1=z;
          z=xyz[y1][x1].pos[2]; if (z0>z) z0=z; if (z1<z) z1=z;
          if (z0   <=0.01) continue;
          if (z1   >=3.90) continue;  // 3.972 pre vsetko nad .=3.95m a 4.000 ak nechyti vobec nic
          if (z1-z0>=0.10) continue;
          glColor4ubv((BYTE* )&xyz[y0][x0].rgb);
          glVertex3fv((float*)&xyz[y0][x0].pos);
          glColor4ubv((BYTE* )&xyz[y0][x1].rgb);
          glVertex3fv((float*)&xyz[y0][x1].pos);
          glColor4ubv((BYTE* )&xyz[y1][x1].rgb);
          glVertex3fv((float*)&xyz[y1][x1].pos);
          glColor4ubv((BYTE* )&xyz[y1][x0].rgb);
          glVertex3fv((float*)&xyz[y1][x0].pos);
          }
      glEnd();
  

  OpenGL .. . :

  

• , ofsx,ofsy copy_images(). . 1 , copy_images . , :

  

  , +17 x +4 y. , :

  

,

Answer 2

, . , . , , - . mathworks fex, : ginputc.

matlab :

clc; clear all; close all;

% no of keypoint 
N = 7; 

I  = imread('2.jpg');
I = rgb2gray(I);
[Gx, Gy] = imgradientxy(I, 'Sobel');
[Gmag, ~] = imgradient(Gx, Gy);
figure, imshow(Gmag, [ ]), title('Gradient magnitude')
I = Gmag;

[x,y] = ginputc(N, 'Color' , 'r');
matchedpoint1 = [x y];

J = imread('2.png');
[Gx, Gy] = imgradientxy(J, 'Sobel');
[Gmag, ~] = imgradient(Gx, Gy);
figure, imshow(Gmag, [ ]), title('Gradient magnitude')
J = Gmag;

[x, y] = ginputc(N, 'Color' , 'r');
matchedpoint2 = [x y];    

[tform,inlierPtsDistorted,inlierPtsOriginal] = estimateGeometricTransform(matchedpoint2,matchedpoint1,'similarity');
figure; showMatchedFeatures(J,I,inlierPtsOriginal,inlierPtsDistorted);
title('Matched inlier points');

I = imread('2.jpg'); J = imread('2.png');
I = rgb2gray(I);
outputView = imref2d(size(I));
Ir = imwarp(J,tform,'OutputView',outputView);
figure; imshow(Ir, []);
title('Recovered image');    

figure,imshowpair(I,J,'diff'),title('Difference with original');
figure,imshowpair(I,Ir,'diff'),title('Difference with restored');

1

- , rgb, . . :

2

ginput ginputc, . . SURF, .

3

estimategeometrictransform tform, , . .

, , . , @Specktre . , . , vap rgb-d-t dataset.