Image scaling (KeepAspectRatioByExpanding) via OpenGL

Question

I am trying to implement image scaling in OpenGL using only glTexCoord2f() and glVertex2f() .

Let me explain: after loading QImage and sending it to gpu using glTexImage2D() I need to perform image scaling operations based on the Qt specification . Qt defines these 3 operations (see Figure below):

I think this is the only way to do this, since my application is a Qt plugin, and this task should be done inside the paint() method of the class. The IgnoreAspectRatio operation IgnoreAspectRatio quite simple and works right now. KeepAspectRatio gave me some problems initially, but now it also works. Unfortunately, KeepAspectRatioByExpanding gives me headaches .

I am sharing what I have done so far, and I appreciate your help on this issue:

main.cpp:

 #include "oglWindow.h" #include <QtGui/QApplication> int main(int argc, char *argv[]) { QApplication a(argc, argv); oglWindow w; w.show(); return a.exec(); } 

oglWindow.cpp:

 #include "oglWindow.h" #include "glwidget.h" #include <QGridLayout> oglWindow::oglWindow(QWidget *parent, Qt::WFlags flags) : QMainWindow(parent, flags) { ui.setupUi(this); GLWidget *openGL = new GLWidget(this); QGridLayout *layout = new QGridLayout; setLayout(layout); } oglWindow::~oglWindow() { } 

oglWindow.h:

 #ifndef oglWindow_H #define oglWindow_H #include <QtGui/QMainWindow> #include "ui_yuv_to_rgb.h" class oglWindow : public QMainWindow { Q_OBJECT public: oglWindow(QWidget *parent = 0, Qt::WFlags flags = 0); ~oglWindow(); private: Ui::oglWindowClass ui; }; #endif // oglWindow_H 

glwidget.cpp:

 #ifdef _MSC_VER #include <windows.h> #include <GL/glew.h> #include <GL/gl.h> #else #include <GL/gl.h> #endif #include "glwidget.h" #include <QDebug> #include <iostream> #include <fstream> #include <assert.h> static const char *p_s_fragment_shader = "#extension GL_ARB_texture_rectangle : enable\n" "uniform sampler2DRect tex;" "uniform float ImgHeight, chromaHeight_Half, chromaWidth;" "void main()" "{" " vec2 t = gl_TexCoord[0].xy;" // get texcoord from fixed-function pipeline " float CbY = ImgHeight + floor(ty / 4.0);" " float CrY = ImgHeight + chromaHeight_Half + floor(ty / 4.0);" " float CbCrX = floor(tx / 2.0) + chromaWidth * floor(mod(ty, 2.0));" " float Cb = texture2DRect(tex, vec2(CbCrX, CbY)).x - .5;" " float Cr = texture2DRect(tex, vec2(CbCrX, CrY)).x - .5;" " float y = texture2DRect(tex, t).x;" // redundant texture read optimized away by texture cache " float r = y + 1.28033 * Cr;" " float g = y - .21482 * Cb - .38059 * Cr;" " float b = y + 2.12798 * Cb;" " gl_FragColor = vec4(r, g, b, 1.0);" "}"; GLWidget::GLWidget(QWidget *parent) : QGLWidget(QGLFormat(QGL::SampleBuffers), parent), _frame(NULL) { setAutoFillBackground(false); setMinimumSize(640, 480); /* Load 1280x768 YV12 frame from the disk */ _frame = new QImage(1280, 768, QImage::Format_RGB888); if (!_frame) { qDebug() << "> GLWidget::GLWidget !!! Failed to create _frame"; return; } std::ifstream yuv_file("bloco.yv12", std::ios::in | std::ios::binary | std::ios::ate); if (!yuv_file.is_open()) { qDebug() << "> GLWidget::GLWidget !!! Failed to load yuv file"; return; } int yuv_file_sz = yuv_file.tellg(); unsigned char* memblock = new unsigned char[yuv_file_sz]; if (!memblock) { qDebug() << "> GLWidget::GLWidget !!! Failed to allocate memblock"; return; } yuv_file.seekg(0, std::ios::beg); yuv_file.read((char*)memblock, yuv_file_sz); yuv_file.close(); qMemCopy(_frame->scanLine(0), memblock, yuv_file_sz); delete[] memblock; } GLWidget::~GLWidget() { if (_frame) delete _frame; } void GLWidget::paintEvent(QPaintEvent *event) { QPainter painter(this); painter.setRenderHint(QPainter::Antialiasing); qDebug() << "> GLWidget::paintEvent OpenGL:" << ((painter.paintEngine()->type() != QPaintEngine::OpenGL && painter.paintEngine()->type() != QPaintEngine::OpenGL2) ? "disabled" : "enabled"); QGLContext* context = const_cast<QGLContext *>(QGLContext::currentContext()); if (!context) { qDebug() << "> GLWidget::paintEvent !!! Unable to retrieve OGL context"; return; } context->makeCurrent(); painter.fillRect(QRectF(QPoint(0, 0), QSize(1280, 768)), Qt::black); painter.beginNativePainting(); /* Initialize GL extensions */ GLenum err = glewInit(); if (err != GLEW_OK) { qDebug() << "> GLWidget::paintEvent !!! glewInit failed with: " << err; return; } if (!GLEW_VERSION_2_1) // check that the machine supports the 2.1 API. { qDebug() << "> GLWidget::paintEvent !!! System doesn't support GLEW_VERSION_2_1"; return; } /* Setting up texture and transfering data to the GPU */ static GLuint texture = 0; if (texture != 0) { context->deleteTexture(texture); } glPixelStorei(GL_UNPACK_ALIGNMENT, 1); glBindTexture(GL_TEXTURE_RECTANGLE_ARB, texture); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, GL_LUMINANCE, _frame->width(), _frame->height() + _frame->height() / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, _frame->bits()); assert(glGetError() == GL_NO_ERROR); glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE); glEnable(GL_TEXTURE_RECTANGLE_ARB); glClearColor(0.3, 0.3, 0.4, 1.0); int img_width = _frame->width(); int img_height = _frame->height(); int offset_x = 0; int offset_y = 0; GLfloat gl_width = width(); // GL context size GLfloat gl_height = height(); /* Initialize shaders and execute them */ _init_shaders(); qDebug() << "paint(): gl_width:" << gl_width << " gl_height:" << gl_height << " img:" << _frame->width() << "x" << _frame->height(); int fill_mode = 1; switch (fill_mode) { case 0: // KeepAspectRatioByExpanding { // need help! } break; case 1: // IgnoreAspectRatio { // Nothing special needs to be done for this operation. } break; case 2: // KeepAspectRatio default: { // Compute aspect ratio and offset Y for widescreen borders double ratiox = img_width/gl_width; double ratioy = img_height/gl_height; if (ratiox > ratioy) { gl_height = qRound(img_height / ratiox); offset_y = qRound((height() - gl_height) / 2); gl_height += offset_y * 2; } else { gl_width = qRound(img_width / ratioy); offset_x = qRound((width() - gl_width) / 2); gl_width += offset_x * 2; } } break; } // Mirroring texture coordinates to flip the image vertically glBegin(GL_QUADS); glTexCoord2f(0.0f, img_height); glVertex2f(offset_x, gl_height - offset_y); glTexCoord2f(img_width, img_height); glVertex2f(gl_width - offset_x, gl_height - offset_y); glTexCoord2f(img_width, 0.0f); glVertex2f(gl_width - offset_x, offset_y); glTexCoord2f(0.0f, 0.0f); glVertex2f(offset_x, offset_y); glEnd(); painter.endNativePainting(); } void GLWidget::_init_shaders() { int f = glCreateShader(GL_FRAGMENT_SHADER); glShaderSource(f, 1, &p_s_fragment_shader, 0); glCompileShader(f); _shader_program = glCreateProgram(); glAttachShader(_shader_program, f); glLinkProgram(_shader_program); glUseProgram(_shader_program); glUniform1i(glGetUniformLocation(_shader_program, "tex"), 0); glUniform1f(glGetUniformLocation(_shader_program, "ImgHeight"), _frame->height()); glUniform1f(glGetUniformLocation(_shader_program, "chromaHeight_Half"), (_frame->height() / 2) / 2); glUniform1f(glGetUniformLocation(_shader_program, "chromaWidth"), _frame->width() / 2); } 

glwidget.h

 #include <QtOpenGL/QGLWidget> #include <QtGui/QImage> #include <QPainter> class GLWidget : public QGLWidget { Q_OBJECT public: GLWidget(QWidget *parent = 0); ~GLWidget(); void paintEvent(QPaintEvent *event); private: void _init_shaders(); bool _checkShader(int n_shader_object); QImage* _frame; int _shader_program; }; 

And here you can download the data file .

Answer 1

You can simply copy the “keep aspect ratio” branch (provided that it works) and simply flip the relationship comparison sign, i.e.:

 if (ratiox > ratioy) 

becomes

 if (ratiox <= ratioy) 

But I'm not sure that it actually works (the computational relationship always beat me - and yours is complicated), and I don't have Qt atm, so I can’t try. But that should do it. Please note that the image will be centered (does not align to the left, as in your image), but this can be easily fixed.

EDIT

Here is the source code that works in the GLUT application (no QT, sorry):

 static void DrawObject(void) { int img_width = 1280;//_frame->width(); int img_height = 720;//_frame->height(); GLfloat offset_x = -1; GLfloat offset_y = -1; int p_viewport[4]; glGetIntegerv(GL_VIEWPORT, p_viewport); // don't have QT :'( GLfloat gl_width = p_viewport[2];//width(); // GL context size GLfloat gl_height = p_viewport[3];//height(); int n_mode = 0; switch(n_mode) { case 0: // KeepAspectRatioByExpanding { float ratioImg = float(img_width) / img_height; float ratioScreen = gl_width / gl_height; if(ratioImg < ratioScreen) { gl_width = 2; gl_height = 2 * ratioScreen / ratioImg; } else { gl_height = 2; gl_width = 2 / ratioScreen * ratioImg; } // calculate image size } break; case 1: // IgnoreAspectRatio gl_width = 2; gl_height = 2; // OpenGL normalized coordinates are -1 to +1 .. hence width (or height) = +1 - (-1) = 2 break; case 2: // KeepAspectRatio { float ratioImg = float(img_width) / img_height; float ratioScreen = gl_width / gl_height; if(ratioImg > ratioScreen) { gl_width = 2; gl_height = 2 * ratioScreen / ratioImg; } else { gl_height = 2; gl_width = 2 / ratioScreen * ratioImg; } // calculate image size offset_x = -1 + (2 - gl_width) * .5f; offset_y = -1 + (2 - gl_height) * .5f; // center on screen } break; } glMatrixMode(GL_PROJECTION); glLoadIdentity(); glMatrixMode( GL_MODELVIEW ); glLoadIdentity(); // just simple ortho view, no fancy transform ... glBegin(GL_QUADS); glTexCoord2f(0, 0); glVertex2f(offset_x, offset_y); glTexCoord2f(ImgWidth, 0); glVertex2f(offset_x + gl_width, offset_y); glTexCoord2f(ImgWidth, ImgHeight); glVertex2f(offset_x + gl_width, offset_y + gl_height); glTexCoord2f(0, ImgHeight); glVertex2f(offset_x, offset_y + gl_height); glEnd(); // draw a single quad } 

This works by comparing the ratio of screen to image format. In fact, you are comparing the ratio of image width to screen width with image height to screen height. It is, at least, suspicious, not to say wrong.

In addition, the normalized OpenGL coordinates (subject to a simple orthogonal representation) are in the range (-1, -1) for the lower left corner to (1, 1) for the upper right. This means that the normalized width and height are 2, and the offset is (-1, -1). The rest of the code should be clear. If the texture is upside down (I tested with some general texture, not sure if it was vertical), just change the coordinates of the texture in the appropriate direction (swap 0s for ImgWidth (or height) and vice versa).

EDIT2

Using pixel coordinates (without using the normalized OpenGL coordinates) is even easier. You can use:

 static void DrawObject(void) { int img_width = 1280;//_frame->width(); int img_height = 720;//_frame->height(); GLfloat offset_x = 0; GLfloat offset_y = 0; int p_viewport[4]; glGetIntegerv(GL_VIEWPORT, p_viewport); GLfloat gl_width = p_viewport[2];//width(); // GL context size GLfloat gl_height = p_viewport[3];//height(); int n_mode = 0; switch(n_mode) { case 0: // KeepAspectRatioByExpanding { float ratioImg = float(img_width) / img_height; float ratioScreen = gl_width / gl_height; if(ratioImg < ratioScreen) gl_height = gl_width / ratioImg; else gl_width = gl_height * ratioImg; // calculate image size } break; case 1: // IgnoreAspectRatio break; case 2: // KeepAspectRatio { float ratioImg = float(img_width) / img_height; float ratioScreen = gl_width / gl_height; GLfloat orig_width = gl_width; GLfloat orig_height = gl_height; // remember those to be able to center the quad on screen if(ratioImg > ratioScreen) gl_height = gl_width / ratioImg; else gl_width = gl_height * ratioImg; // calculate image size offset_x = 0 + (orig_width - gl_width) * .5f; offset_y = 0 + (orig_height - gl_height) * .5f; // center on screen } break; } glMatrixMode(GL_PROJECTION); glLoadIdentity(); glMatrixMode(GL_MODELVIEW); glLoadIdentity(); glTranslatef(-1, -1, 0); glScalef(2.0f / p_viewport[2], 2.0f / p_viewport[3], 1.0); // just simple ortho view for vertex coordinate to pixel matching glBegin(GL_QUADS); glTexCoord2f(0, 0); glVertex2f(offset_x, offset_y); glTexCoord2f(img_width, 0); glVertex2f(offset_x + gl_width, offset_y); glTexCoord2f(img_width, img_height); glVertex2f(offset_x + gl_width, offset_y + gl_height); glTexCoord2f(0, img_height); glVertex2f(offset_x, offset_y + gl_height); glEnd(); // draw a single quad } 

Note that both versions of the code use NPOT textures. To adapt the code to your object, you need to do something like this:

 void GLWidget::paintEvent(QPaintEvent *event) { QPainter painter(this); painter.setRenderHint(QPainter::Antialiasing); qDebug() << "> GLWidget::paintEvent OpenGL:" << ((painter.paintEngine()->type() != QPaintEngine::OpenGL && painter.paintEngine()->type() != QPaintEngine::OpenGL2) ? "disabled" : "enabled"); QGLContext* context = const_cast<QGLContext *>(QGLContext::currentContext()); if (!context) { qDebug() << "> GLWidget::paintEvent !!! Unable to retrieve OGL context"; return; } context->makeCurrent(); painter.fillRect(QRectF(QPoint(0, 0), QSize(1280, 768)), Qt::black); painter.beginNativePainting(); /* Initialize GL extensions */ GLenum err = glewInit(); if (err != GLEW_OK) { qDebug() << "> GLWidget::paintEvent !!! glewInit failed with: " << err; return; } if (!GLEW_VERSION_2_1) // check that the machine supports the 2.1 API. { qDebug() << "> GLWidget::paintEvent !!! System doesn't support GLEW_VERSION_2_1"; return; } /* Setting up texture and transfering data to the GPU */ static GLuint texture = 0; if (texture != 0) { context->deleteTexture(texture); } glPixelStorei(GL_UNPACK_ALIGNMENT, 1); glBindTexture(GL_TEXTURE_RECTANGLE_ARB, texture); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, GL_LUMINANCE, _frame->width(), _frame->height() + _frame->height() / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, _frame->bits()); assert(glGetError() == GL_NO_ERROR); glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE); glEnable(GL_TEXTURE_RECTANGLE_ARB); glClearColor(0.3, 0.3, 0.4, 1.0); /* Initialize shaders and execute them */ _init_shaders(); int img_width = _frame->width(); int img_height = _frame->height(); GLfloat offset_x = 0; GLfloat offset_y = 0; GLfloat gl_width = width(); // GL context size GLfloat gl_height = height(); qDebug() << "paint(): gl_width:" << gl_width << " gl_height:" << gl_height << " img:" << _frame->width() << "x" << _frame->height(); int fill_mode = 0; switch(fill_mode) { case 0: // KeepAspectRatioByExpanding { float ratioImg = float(img_width) / img_height; float ratioScreen = gl_width / gl_height; if(ratioImg < ratioScreen) gl_height = gl_width / ratioImg; else gl_width = gl_height * ratioImg; // calculate image size } break; case 1: // IgnoreAspectRatio break; case 2: // KeepAspectRatio { float ratioImg = float(img_width) / img_height; float ratioScreen = gl_width / gl_height; GLfloat orig_width = gl_width; GLfloat orig_height = gl_height; // remember those to be able to center the quad on screen if(ratioImg > ratioScreen) gl_height = gl_width / ratioImg; else gl_width = gl_height * ratioImg; // calculate image size offset_x = 0 + (orig_width - gl_width) * .5f; offset_y = 0 + (orig_height - gl_height) * .5f; // center on screen } break; } glDisable(GL_CULL_FACE); // might cause problems if enabled glBegin(GL_QUADS); glTexCoord2f(0, 0); glVertex2f(offset_x, offset_y); glTexCoord2f(img_width, 0); glVertex2f(offset_x + gl_width, offset_y); glTexCoord2f(img_width, img_height); glVertex2f(offset_x + gl_width, offset_y + gl_height); glTexCoord2f(0, img_height); glVertex2f(offset_x, offset_y + gl_height); glEnd(); // draw a single quad painter.endNativePainting(); } 

There is no way to guarantee that this last piece of code is error free, since I don't have QT. But if there are typos, it should be pretty easy to fix them.

Answer 2

Just do the same math as with KeepAspectRatio, but this time keep the height instead of the width. You will get a width greater than 1, so you have to use the inverse coordinate of the texture for it.

---

The normalized yes / no coordinates will not matter, you just need to add these factors to the texture coordinates. For this, I assume that the image fills the entire texture (from (0,0) to (1,1), which makes it easy to multiply the values ​​with your width / height). Texture wrapper should be disabled.

Default texture coordinates:

 (0,0)-(1,0) | | (0,1)-(1,1) 

Aspect ratio:

 tex_ar = tex_width / tex_height; // aspect ratio for the texture being used scr_ar = scr_width / scr_height; // aspect ratio for the quad being drawn 

KeepAspectRatio (tap from the inside):

 (0, 0)-----------------------(max(1, scr_ar / tex_ar), 0) | | (0, max(1, tex_ar / scr_ar))-(max(1, scr_ / tex_ar), max(1, tex_ar / scr_ar)) 

KeepAspectRatioByExpanding (tap from the outside, just replace max() with min() ):

 (0, 0)-----------------------(min(1, scr_ar / tex_ar), 0) | | (0, min(1, tex_ar / scr_ar))-(min(1, scr_ / tex_ar), min(1, tex_ar / scr_ar)) 

For your case, you just need to multiply the resulting texture coordinates by your width / height.