From 9db6ce0ba499eba53db236558d783a4ff7aa3896 Mon Sep 17 00:00:00 2001 From: Show Liu Date: Fri, 11 Sep 2020 16:55:13 +0800 Subject: qcam: Add ViewFinderGL class to accelerate the format conversion The viewfinderGL accelerates the format conversion by using OpenGL ES shader. The minimum Qt version is bumped to v5.4, as QOpenGLWidget wasn't available before that. Signed-off-by: Show Liu Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- meson.build | 1 + src/qcam/meson.build | 17 +- src/qcam/viewfinder_gl.cpp | 451 +++++++++++++++++++++++++++++++++++++++++++++ src/qcam/viewfinder_gl.h | 96 ++++++++++ 4 files changed, 563 insertions(+), 2 deletions(-) create mode 100644 src/qcam/viewfinder_gl.cpp create mode 100644 src/qcam/viewfinder_gl.h diff --git a/meson.build b/meson.build index 1ea35e92..c58d458c 100644 --- a/meson.build +++ b/meson.build @@ -26,6 +26,7 @@ libcamera_version = libcamera_git_version.split('+')[0] # Configure the build environment. cc = meson.get_compiler('c') +cxx = meson.get_compiler('cpp') config_h = configuration_data() if cc.has_header_symbol('execinfo.h', 'backtrace') diff --git a/src/qcam/meson.build b/src/qcam/meson.build index a4bad0a0..9bb48c0d 100644 --- a/src/qcam/meson.build +++ b/src/qcam/meson.build @@ -16,14 +16,14 @@ qcam_moc_headers = files([ qcam_resources = files([ 'assets/feathericons/feathericons.qrc', - 'assets/shader/shaders.qrc' ]) qt5 = import('qt5') qt5_dep = dependency('qt5', method : 'pkg-config', modules : ['Core', 'Gui', 'Widgets'], - required : get_option('qcam')) + required : get_option('qcam'), + version : '>=5.4') if qt5_dep.found() qcam_deps = [ @@ -42,6 +42,19 @@ if qt5_dep.found() ]) endif + if cxx.has_header_symbol('QOpenGLWidget', 'QOpenGLWidget', + dependencies : qt5_dep, args : '-fPIC') + qcam_sources += files([ + 'viewfinder_gl.cpp', + ]) + qcam_moc_headers += files([ + 'viewfinder_gl.h', + ]) + qcam_resources += files([ + 'assets/shader/shaders.qrc' + ]) + endif + # gcc 9 introduced a deprecated-copy warning that is triggered by Qt until # Qt 5.13. clang 10 introduced the same warning, but detects more issues # that are not fixed in Qt yet. Disable the warning manually in both cases. diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp new file mode 100644 index 00000000..fbe21dcf --- /dev/null +++ b/src/qcam/viewfinder_gl.cpp @@ -0,0 +1,451 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * viewfinderGL.cpp - OpenGL Viewfinder for rendering by OpenGL shader + */ + +#include "viewfinder_gl.h" + +#include + +#include + +static const QList supportedFormats{ + libcamera::formats::NV12, + libcamera::formats::NV21, + libcamera::formats::NV16, + libcamera::formats::NV61, + libcamera::formats::NV24, + libcamera::formats::NV42, + libcamera::formats::YUV420, + libcamera::formats::YVU420, +}; + +ViewFinderGL::ViewFinderGL(QWidget *parent) + : QOpenGLWidget(parent), buffer_(nullptr), yuvData_(nullptr), + fragmentShader_(nullptr), vertexShader_(nullptr), + vertexBuffer_(QOpenGLBuffer::VertexBuffer), + textureU_(QOpenGLTexture::Target2D), + textureV_(QOpenGLTexture::Target2D), + textureY_(QOpenGLTexture::Target2D) +{ +} + +ViewFinderGL::~ViewFinderGL() +{ + removeShader(); +} + +const QList &ViewFinderGL::nativeFormats() const +{ + return supportedFormats; +} + +int ViewFinderGL::setFormat(const libcamera::PixelFormat &format, + const QSize &size) +{ + /* If the fragment is created remove it and create a new one. */ + if (fragmentShader_) { + if (shaderProgram_.isLinked()) { + shaderProgram_.release(); + shaderProgram_.removeShader(fragmentShader_); + delete fragmentShader_; + } + } + + if (!selectFormat(format)) + return -1; + + format_ = format; + size_ = size; + + updateGeometry(); + return 0; +} + +void ViewFinderGL::stop() +{ + if (buffer_) { + renderComplete(buffer_); + buffer_ = nullptr; + } +} + +QImage ViewFinderGL::getCurrentImage() +{ + QMutexLocker locker(&mutex_); + + return grabFramebuffer(); +} + +void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map) +{ + if (buffer->planes().size() != 1) { + qWarning() << "Multi-planar buffers are not supported"; + return; + } + + if (buffer_) + renderComplete(buffer_); + + yuvData_ = static_cast(map->memory); + update(); + buffer_ = buffer; +} + +bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) +{ + bool ret = true; + switch (format) { + case libcamera::formats::NV12: + horzSubSample_ = 2; + vertSubSample_ = 2; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl"; + break; + case libcamera::formats::NV21: + horzSubSample_ = 2; + vertSubSample_ = 2; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl"; + break; + case libcamera::formats::NV16: + horzSubSample_ = 2; + vertSubSample_ = 1; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl"; + break; + case libcamera::formats::NV61: + horzSubSample_ = 2; + vertSubSample_ = 1; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl"; + break; + case libcamera::formats::NV24: + horzSubSample_ = 1; + vertSubSample_ = 1; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl"; + break; + case libcamera::formats::NV42: + horzSubSample_ = 1; + vertSubSample_ = 1; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl"; + break; + case libcamera::formats::YUV420: + horzSubSample_ = 2; + vertSubSample_ = 2; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_3_planes_f.glsl"; + break; + case libcamera::formats::YVU420: + horzSubSample_ = 2; + vertSubSample_ = 2; + vertexShaderSrc_ = ":NV_vertex_shader.glsl"; + fragmentShaderSrc_ = ":NV_3_planes_f.glsl"; + break; + default: + ret = false; + qWarning() << "[ViewFinderGL]:" + << "format not supported."; + break; + }; + + return ret; +} + +bool ViewFinderGL::createVertexShader() +{ + /* Create Vertex Shader */ + vertexShader_ = new QOpenGLShader(QOpenGLShader::Vertex, this); + + /* Compile the vertex shader */ + if (!vertexShader_->compileSourceFile(vertexShaderSrc_)) { + qWarning() << "[ViewFinderGL]:" << vertexShader_->log(); + return false; + } + + shaderProgram_.addShader(vertexShader_); + return true; +} + +bool ViewFinderGL::createFragmentShader() +{ + int attributeVertex; + int attributeTexture; + + /* Create Fragment Shader */ + fragmentShader_ = new QOpenGLShader(QOpenGLShader::Fragment, this); + + /* Compile the fragment shader */ + if (!fragmentShader_->compileSourceFile(fragmentShaderSrc_)) { + qWarning() << "[ViewFinderGL]:" << fragmentShader_->log(); + return false; + } + + shaderProgram_.addShader(fragmentShader_); + + /* Link shader pipeline */ + if (!shaderProgram_.link()) { + qWarning() << "[ViewFinderGL]:" << shaderProgram_.log(); + close(); + } + + /* Bind shader pipeline for use */ + if (!shaderProgram_.bind()) { + qWarning() << "[ViewFinderGL]:" << shaderProgram_.log(); + close(); + } + + attributeVertex = shaderProgram_.attributeLocation("vertexIn"); + attributeTexture = shaderProgram_.attributeLocation("textureIn"); + + shaderProgram_.enableAttributeArray(attributeVertex); + shaderProgram_.setAttributeBuffer(attributeVertex, + GL_FLOAT, + 0, + 2, + 2 * sizeof(GLfloat)); + + shaderProgram_.enableAttributeArray(attributeTexture); + shaderProgram_.setAttributeBuffer(attributeTexture, + GL_FLOAT, + 8 * sizeof(GLfloat), + 2, + 2 * sizeof(GLfloat)); + + textureUniformY_ = shaderProgram_.uniformLocation("tex_y"); + textureUniformU_ = shaderProgram_.uniformLocation("tex_u"); + textureUniformV_ = shaderProgram_.uniformLocation("tex_v"); + + if (!textureY_.isCreated()) + textureY_.create(); + + if (!textureU_.isCreated()) + textureU_.create(); + + if (!textureV_.isCreated()) + textureV_.create(); + + id_y_ = textureY_.textureId(); + id_u_ = textureU_.textureId(); + id_v_ = textureV_.textureId(); + return true; +} + +void ViewFinderGL::configureTexture(unsigned int id) +{ + glBindTexture(GL_TEXTURE_2D, id); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +} + +void ViewFinderGL::removeShader() +{ + if (shaderProgram_.isLinked()) { + shaderProgram_.release(); + shaderProgram_.removeAllShaders(); + } + + if (fragmentShader_) + delete fragmentShader_; + + if (vertexShader_) + delete vertexShader_; +} + +void ViewFinderGL::initializeGL() +{ + initializeOpenGLFunctions(); + glEnable(GL_TEXTURE_2D); + glDisable(GL_DEPTH_TEST); + + static const GLfloat coordinates[2][4][2]{ + { + /* Vertex coordinates */ + { -1.0f, -1.0f }, + { -1.0f, +1.0f }, + { +1.0f, +1.0f }, + { +1.0f, -1.0f }, + }, + { + /* Texture coordinates */ + { 0.0f, 1.0f }, + { 0.0f, 0.0f }, + { 1.0f, 0.0f }, + { 1.0f, 1.0f }, + }, + }; + + vertexBuffer_.create(); + vertexBuffer_.bind(); + vertexBuffer_.allocate(coordinates, sizeof(coordinates)); + + /* Create Vertex Shader */ + if (!createVertexShader()) + qWarning() << "[ViewFinderGL]: create vertex shader failed."; + + glClearColor(1.0f, 1.0f, 1.0f, 0.0f); +} + +void ViewFinderGL::doRender() +{ + switch (format_) { + case libcamera::formats::NV12: + case libcamera::formats::NV21: + case libcamera::formats::NV16: + case libcamera::formats::NV61: + case libcamera::formats::NV24: + case libcamera::formats::NV42: + /* Activate texture Y */ + glActiveTexture(GL_TEXTURE0); + configureTexture(id_y_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width(), + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + yuvData_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + + /* Activate texture UV/VU */ + glActiveTexture(GL_TEXTURE1); + configureTexture(id_u_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RG, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RG, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height()); + shaderProgram_.setUniformValue(textureUniformU_, 1); + break; + + case libcamera::formats::YUV420: + /* Activate texture Y */ + glActiveTexture(GL_TEXTURE0); + configureTexture(id_y_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width(), + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + yuvData_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + + /* Activate texture U */ + glActiveTexture(GL_TEXTURE1); + configureTexture(id_u_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height()); + shaderProgram_.setUniformValue(textureUniformU_, 1); + + /* Activate texture V */ + glActiveTexture(GL_TEXTURE2); + configureTexture(id_v_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height() * 5 / 4); + shaderProgram_.setUniformValue(textureUniformV_, 2); + break; + + case libcamera::formats::YVU420: + /* Activate texture Y */ + glActiveTexture(GL_TEXTURE0); + configureTexture(id_y_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width(), + size_.height(), + 0, + GL_RED, + GL_UNSIGNED_BYTE, + yuvData_); + shaderProgram_.setUniformValue(textureUniformY_, 0); + + /* Activate texture V */ + glActiveTexture(GL_TEXTURE2); + configureTexture(id_v_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height()); + shaderProgram_.setUniformValue(textureUniformV_, 2); + + /* Activate texture U */ + glActiveTexture(GL_TEXTURE1); + configureTexture(id_u_); + glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RED, + size_.width() / horzSubSample_, + size_.height() / vertSubSample_, + 0, + GL_RED, + GL_UNSIGNED_BYTE, + (char *)yuvData_ + size_.width() * size_.height() * 5 / 4); + shaderProgram_.setUniformValue(textureUniformU_, 1); + break; + + default: + break; + }; +} + +void ViewFinderGL::paintGL() +{ + if (!fragmentShader_) + if (!createFragmentShader()) { + qWarning() << "[ViewFinderGL]:" + << "create fragment shader failed."; + } + + if (yuvData_) { + glClearColor(0.0, 0.0, 0.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + + doRender(); + glDrawArrays(GL_TRIANGLE_FAN, 0, 4); + } +} + +void ViewFinderGL::resizeGL(int w, int h) +{ + glViewport(0, 0, w, h); +} + +QSize ViewFinderGL::sizeHint() const +{ + return size_.isValid() ? size_ : QSize(640, 480); +} diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h new file mode 100644 index 00000000..69502b7a --- /dev/null +++ b/src/qcam/viewfinder_gl.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2020, Linaro + * + * viewfinder_GL.h - OpenGL Viewfinder for rendering by OpenGL shader + * + */ +#ifndef __VIEWFINDER_GL_H__ +#define __VIEWFINDER_GL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "viewfinder.h" + +class ViewFinderGL : public QOpenGLWidget, + public ViewFinder, + protected QOpenGLFunctions +{ + Q_OBJECT + +public: + ViewFinderGL(QWidget *parent = nullptr); + ~ViewFinderGL(); + + const QList &nativeFormats() const override; + + int setFormat(const libcamera::PixelFormat &format, const QSize &size) override; + void render(libcamera::FrameBuffer *buffer, MappedBuffer *map) override; + void stop() override; + + QImage getCurrentImage() override; + +Q_SIGNALS: + void renderComplete(libcamera::FrameBuffer *buffer); + +protected: + void initializeGL() override; + void paintGL() override; + void resizeGL(int w, int h) override; + QSize sizeHint() const override; + +private: + bool selectFormat(const libcamera::PixelFormat &format); + + void configureTexture(unsigned int id); + bool createFragmentShader(); + bool createVertexShader(); + void removeShader(); + void doRender(); + + /* Captured image size, format and buffer */ + libcamera::FrameBuffer *buffer_; + libcamera::PixelFormat format_; + QSize size_; + unsigned char *yuvData_; + + /* OpenGL components for rendering */ + QOpenGLShader *fragmentShader_; + QOpenGLShader *vertexShader_; + QOpenGLShaderProgram shaderProgram_; + + /* Vertex buffer */ + QOpenGLBuffer vertexBuffer_; + + /* Fragment and Vertex shader file name */ + QString fragmentShaderSrc_; + QString vertexShaderSrc_; + + /* YUV texture planars and parameters */ + GLuint id_u_; + GLuint id_v_; + GLuint id_y_; + GLuint textureUniformU_; + GLuint textureUniformV_; + GLuint textureUniformY_; + QOpenGLTexture textureU_; + QOpenGLTexture textureV_; + QOpenGLTexture textureY_; + unsigned int horzSubSample_; + unsigned int vertSubSample_; + + QMutex mutex_; /* Prevent concurrent access to image_ */ +}; + +#endif /* __VIEWFINDER_GL_H__ */ -- cgit v1.2.1