qcam: Add ViewFinderGL class to accelerate the format conversion

The viewfinderGL accelerates the format conversion by using OpenGL ES
shader.

The minimum Qt version is bumped to v5.4, as QOpenGLWidget wasn't
available before that.

Signed-off-by: Show Liu <show.liu@linaro.org>
Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>

3 files changed, 562 insertions, 2 deletions

diff --git a/src/qcam/meson.build b/src/qcam/meson.build
index a4bad0a0..9bb48c0d 100644
--- a/src/qcam/meson.build
+++ b/src/qcam/meson.build
@@ -16,14 +16,14 @@ qcam_moc_headers = files([
 
 qcam_resources = files([
     'assets/feathericons/feathericons.qrc',
-    'assets/shader/shaders.qrc'
 ])
 
 qt5 = import('qt5')
 qt5_dep = dependency('qt5',
                      method : 'pkg-config',
                      modules : ['Core', 'Gui', 'Widgets'],
-                     required : get_option('qcam'))
+                     required : get_option('qcam'),
+                     version : '>=5.4')
 
 if qt5_dep.found()
     qcam_deps = [
@@ -42,6 +42,19 @@ if qt5_dep.found()
         ])
     endif
 
+    if cxx.has_header_symbol('QOpenGLWidget', 'QOpenGLWidget',
+                             dependencies : qt5_dep, args : '-fPIC')
+        qcam_sources += files([
+            'viewfinder_gl.cpp',
+        ])
+        qcam_moc_headers += files([
+            'viewfinder_gl.h',
+        ])
+        qcam_resources += files([
+            'assets/shader/shaders.qrc'
+        ])
+    endif
+
     # gcc 9 introduced a deprecated-copy warning that is triggered by Qt until
     # Qt 5.13. clang 10 introduced the same warning, but detects more issues
     # that are not fixed in Qt yet. Disable the warning manually in both cases.
diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp
new file mode 100644
index 00000000..fbe21dcf
--- /dev/null
+++ b/src/qcam/viewfinder_gl.cpp
@@ -0,0 +1,451 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2020, Linaro
+ *
+ * viewfinderGL.cpp - OpenGL Viewfinder for rendering by OpenGL shader
+ */
+
+#include "viewfinder_gl.h"
+
+#include <QImage>
+
+#include <libcamera/formats.h>
+
+static const QList<libcamera::PixelFormat> supportedFormats{
+	libcamera::formats::NV12,
+	libcamera::formats::NV21,
+	libcamera::formats::NV16,
+	libcamera::formats::NV61,
+	libcamera::formats::NV24,
+	libcamera::formats::NV42,
+	libcamera::formats::YUV420,
+	libcamera::formats::YVU420,
+};
+
+ViewFinderGL::ViewFinderGL(QWidget *parent)
+	: QOpenGLWidget(parent), buffer_(nullptr), yuvData_(nullptr),
+	  fragmentShader_(nullptr), vertexShader_(nullptr),
+	  vertexBuffer_(QOpenGLBuffer::VertexBuffer),
+	  textureU_(QOpenGLTexture::Target2D),
+	  textureV_(QOpenGLTexture::Target2D),
+	  textureY_(QOpenGLTexture::Target2D)
+{
+}
+
+ViewFinderGL::~ViewFinderGL()
+{
+	removeShader();
+}
+
+const QList<libcamera::PixelFormat> &ViewFinderGL::nativeFormats() const
+{
+	return supportedFormats;
+}
+
+int ViewFinderGL::setFormat(const libcamera::PixelFormat &format,
+			    const QSize &size)
+{
+	/* If the fragment is created remove it and create a new one. */
+	if (fragmentShader_) {
+		if (shaderProgram_.isLinked()) {
+			shaderProgram_.release();
+			shaderProgram_.removeShader(fragmentShader_);
+			delete fragmentShader_;
+		}
+	}
+
+	if (!selectFormat(format))
+		return -1;
+
+	format_ = format;
+	size_ = size;
+
+	updateGeometry();
+	return 0;
+}
+
+void ViewFinderGL::stop()
+{
+	if (buffer_) {
+		renderComplete(buffer_);
+		buffer_ = nullptr;
+	}
+}
+
+QImage ViewFinderGL::getCurrentImage()
+{
+	QMutexLocker locker(&mutex_);
+
+	return grabFramebuffer();
+}
+
+void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map)
+{
+	if (buffer->planes().size() != 1) {
+		qWarning() << "Multi-planar buffers are not supported";
+		return;
+	}
+
+	if (buffer_)
+		renderComplete(buffer_);
+
+	yuvData_ = static_cast<unsigned char *>(map->memory);
+	update();
+	buffer_ = buffer;
+}
+
+bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
+{
+	bool ret = true;
+	switch (format) {
+	case libcamera::formats::NV12:
+		horzSubSample_ = 2;
+		vertSubSample_ = 2;
+		vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+		fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl";
+		break;
+	case libcamera::formats::NV21:
+		horzSubSample_ = 2;
+		vertSubSample_ = 2;
+		vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+		fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl";
+		break;
+	case libcamera::formats::NV16:
+		horzSubSample_ = 2;
+		vertSubSample_ = 1;
+		vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+		fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl";
+		break;
+	case libcamera::formats::NV61:
+		horzSubSample_ = 2;
+		vertSubSample_ = 1;
+		vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+		fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl";
+		break;
+	case libcamera::formats::NV24:
+		horzSubSample_ = 1;
+		vertSubSample_ = 1;
+		vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+		fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl";
+		break;
+	case libcamera::formats::NV42:
+		horzSubSample_ = 1;
+		vertSubSample_ = 1;
+		vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+		fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl";
+		break;
+	case libcamera::formats::YUV420:
+		horzSubSample_ = 2;
+		vertSubSample_ = 2;
+		vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+		fragmentShaderSrc_ = ":NV_3_planes_f.glsl";
+		break;
+	case libcamera::formats::YVU420:
+		horzSubSample_ = 2;
+		vertSubSample_ = 2;
+		vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+		fragmentShaderSrc_ = ":NV_3_planes_f.glsl";
+		break;
+	default:
+		ret = false;
+		qWarning() << "[ViewFinderGL]:"
+			   << "format not supported.";
+		break;
+	};
+
+	return ret;
+}
+
+bool ViewFinderGL::createVertexShader()
+{
+	/* Create Vertex Shader */
+	vertexShader_ = new QOpenGLShader(QOpenGLShader::Vertex, this);
+
+	/* Compile the vertex shader */
+	if (!vertexShader_->compileSourceFile(vertexShaderSrc_)) {
+		qWarning() << "[ViewFinderGL]:" << vertexShader_->log();
+		return false;
+	}
+
+	shaderProgram_.addShader(vertexShader_);
+	return true;
+}
+
+bool ViewFinderGL::createFragmentShader()
+{
+	int attributeVertex;
+	int attributeTexture;
+
+	/* Create Fragment Shader */
+	fragmentShader_ = new QOpenGLShader(QOpenGLShader::Fragment, this);
+
+	/* Compile the fragment shader */
+	if (!fragmentShader_->compileSourceFile(fragmentShaderSrc_)) {
+		qWarning() << "[ViewFinderGL]:" << fragmentShader_->log();
+		return false;
+	}
+
+	shaderProgram_.addShader(fragmentShader_);
+
+	/* Link shader pipeline */
+	if (!shaderProgram_.link()) {
+		qWarning() << "[ViewFinderGL]:" << shaderProgram_.log();
+		close();
+	}
+
+	/* Bind shader pipeline for use */
+	if (!shaderProgram_.bind()) {
+		qWarning() << "[ViewFinderGL]:" << shaderProgram_.log();
+		close();
+	}
+
+	attributeVertex = shaderProgram_.attributeLocation("vertexIn");
+	attributeTexture = shaderProgram_.attributeLocation("textureIn");
+
+	shaderProgram_.enableAttributeArray(attributeVertex);
+	shaderProgram_.setAttributeBuffer(attributeVertex,
+					  GL_FLOAT,
+					  0,
+					  2,
+					  2 * sizeof(GLfloat));
+
+	shaderProgram_.enableAttributeArray(attributeTexture);
+	shaderProgram_.setAttributeBuffer(attributeTexture,
+					  GL_FLOAT,
+					  8 * sizeof(GLfloat),
+					  2,
+					  2 * sizeof(GLfloat));
+
+	textureUniformY_ = shaderProgram_.uniformLocation("tex_y");
+	textureUniformU_ = shaderProgram_.uniformLocation("tex_u");
+	textureUniformV_ = shaderProgram_.uniformLocation("tex_v");
+
+	if (!textureY_.isCreated())
+		textureY_.create();
+
+	if (!textureU_.isCreated())
+		textureU_.create();
+
+	if (!textureV_.isCreated())
+		textureV_.create();
+
+	id_y_ = textureY_.textureId();
+	id_u_ = textureU_.textureId();
+	id_v_ = textureV_.textureId();
+	return true;
+}
+
+void ViewFinderGL::configureTexture(unsigned int id)
+{
+	glBindTexture(GL_TEXTURE_2D, id);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+}
+
+void ViewFinderGL::removeShader()
+{
+	if (shaderProgram_.isLinked()) {
+		shaderProgram_.release();
+		shaderProgram_.removeAllShaders();
+	}
+
+	if (fragmentShader_)
+		delete fragmentShader_;
+
+	if (vertexShader_)
+		delete vertexShader_;
+}
+
+void ViewFinderGL::initializeGL()
+{
+	initializeOpenGLFunctions();
+	glEnable(GL_TEXTURE_2D);
+	glDisable(GL_DEPTH_TEST);
+
+	static const GLfloat coordinates[2][4][2]{
+		{
+			/* Vertex coordinates */
+			{ -1.0f, -1.0f },
+			{ -1.0f, +1.0f },
+			{ +1.0f, +1.0f },
+			{ +1.0f, -1.0f },
+		},
+		{
+			/* Texture coordinates */
+			{ 0.0f, 1.0f },
+			{ 0.0f, 0.0f },
+			{ 1.0f, 0.0f },
+			{ 1.0f, 1.0f },
+		},
+	};
+
+	vertexBuffer_.create();
+	vertexBuffer_.bind();
+	vertexBuffer_.allocate(coordinates, sizeof(coordinates));
+
+	/* Create Vertex Shader */
+	if (!createVertexShader())
+		qWarning() << "[ViewFinderGL]: create vertex shader failed.";
+
+	glClearColor(1.0f, 1.0f, 1.0f, 0.0f);
+}
+
+void ViewFinderGL::doRender()
+{
+	switch (format_) {
+	case libcamera::formats::NV12:
+	case libcamera::formats::NV21:
+	case libcamera::formats::NV16:
+	case libcamera::formats::NV61:
+	case libcamera::formats::NV24:
+	case libcamera::formats::NV42:
+		/* Activate texture Y */
+		glActiveTexture(GL_TEXTURE0);
+		configureTexture(id_y_);
+		glTexImage2D(GL_TEXTURE_2D,
+			     0,
+			     GL_RED,
+			     size_.width(),
+			     size_.height(),
+			     0,
+			     GL_RED,
+			     GL_UNSIGNED_BYTE,
+			     yuvData_);
+		shaderProgram_.setUniformValue(textureUniformY_, 0);
+
+		/* Activate texture UV/VU */
+		glActiveTexture(GL_TEXTURE1);
+		configureTexture(id_u_);
+		glTexImage2D(GL_TEXTURE_2D,
+			     0,
+			     GL_RG,
+			     size_.width() / horzSubSample_,
+			     size_.height() / vertSubSample_,
+			     0,
+			     GL_RG,
+			     GL_UNSIGNED_BYTE,
+			     (char *)yuvData_ + size_.width() * size_.height());
+		shaderProgram_.setUniformValue(textureUniformU_, 1);
+		break;
+
+	case libcamera::formats::YUV420:
+		/* Activate texture Y */
+		glActiveTexture(GL_TEXTURE0);
+		configureTexture(id_y_);
+		glTexImage2D(GL_TEXTURE_2D,
+			     0,
+			     GL_RED,
+			     size_.width(),
+			     size_.height(),
+			     0,
+			     GL_RED,
+			     GL_UNSIGNED_BYTE,
+			     yuvData_);
+		shaderProgram_.setUniformValue(textureUniformY_, 0);
+
+		/* Activate texture U */
+		glActiveTexture(GL_TEXTURE1);
+		configureTexture(id_u_);
+		glTexImage2D(GL_TEXTURE_2D,
+			     0,
+			     GL_RED,
+			     size_.width() / horzSubSample_,
+			     size_.height() / vertSubSample_,
+			     0,
+			     GL_RED,
+			     GL_UNSIGNED_BYTE,
+			     (char *)yuvData_ + size_.width() * size_.height());
+		shaderProgram_.setUniformValue(textureUniformU_, 1);
+
+		/* Activate texture V */
+		glActiveTexture(GL_TEXTURE2);
+		configureTexture(id_v_);
+		glTexImage2D(GL_TEXTURE_2D,
+			     0,
+			     GL_RED,
+			     size_.width() / horzSubSample_,
+			     size_.height() / vertSubSample_,
+			     0,
+			     GL_RED,
+			     GL_UNSIGNED_BYTE,
+			     (char *)yuvData_ + size_.width() * size_.height() * 5 / 4);
+		shaderProgram_.setUniformValue(textureUniformV_, 2);
+		break;
+
+	case libcamera::formats::YVU420:
+		/* Activate texture Y */
+		glActiveTexture(GL_TEXTURE0);
+		configureTexture(id_y_);
+		glTexImage2D(GL_TEXTURE_2D,
+			     0,
+			     GL_RED,
+			     size_.width(),
+			     size_.height(),
+			     0,
+			     GL_RED,
+			     GL_UNSIGNED_BYTE,
+			     yuvData_);
+		shaderProgram_.setUniformValue(textureUniformY_, 0);
+
+		/* Activate texture V */
+		glActiveTexture(GL_TEXTURE2);
+		configureTexture(id_v_);
+		glTexImage2D(GL_TEXTURE_2D,
+			     0,
+			     GL_RED,
+			     size_.width() / horzSubSample_,
+			     size_.height() / vertSubSample_,
+			     0,
+			     GL_RED,
+			     GL_UNSIGNED_BYTE,
+			     (char *)yuvData_ + size_.width() * size_.height());
+		shaderProgram_.setUniformValue(textureUniformV_, 2);
+
+		/* Activate texture U */
+		glActiveTexture(GL_TEXTURE1);
+		configureTexture(id_u_);
+		glTexImage2D(GL_TEXTURE_2D,
+			     0,
+			     GL_RED,
+			     size_.width() / horzSubSample_,
+			     size_.height() / vertSubSample_,
+			     0,
+			     GL_RED,
+			     GL_UNSIGNED_BYTE,
+			     (char *)yuvData_ + size_.width() * size_.height() * 5 / 4);
+		shaderProgram_.setUniformValue(textureUniformU_, 1);
+		break;
+
+	default:
+		break;
+	};
+}
+
+void ViewFinderGL::paintGL()
+{
+	if (!fragmentShader_)
+		if (!createFragmentShader()) {
+			qWarning() << "[ViewFinderGL]:"
+				   << "create fragment shader failed.";
+		}
+
+	if (yuvData_) {
+		glClearColor(0.0, 0.0, 0.0, 1.0);
+		glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+
+		doRender();
+		glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
+	}
+}
+
+void ViewFinderGL::resizeGL(int w, int h)
+{
+	glViewport(0, 0, w, h);
+}
+
+QSize ViewFinderGL::sizeHint() const
+{
+	return size_.isValid() ? size_ : QSize(640, 480);
+}
diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h
new file mode 100644
index 00000000..69502b7a
--- /dev/null
+++ b/src/qcam/viewfinder_gl.h
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2020, Linaro
+ *
+ * viewfinder_GL.h - OpenGL Viewfinder for rendering by OpenGL shader
+ *
+ */
+#ifndef __VIEWFINDER_GL_H__
+#define __VIEWFINDER_GL_H__
+
+#include <QImage>
+#include <QMutex>
+#include <QOpenGLBuffer>
+#include <QOpenGLFunctions>
+#include <QOpenGLShader>
+#include <QOpenGLShaderProgram>
+#include <QOpenGLTexture>
+#include <QOpenGLWidget>
+#include <QSize>
+
+#include <libcamera/buffer.h>
+#include <libcamera/formats.h>
+
+#include "viewfinder.h"
+
+class ViewFinderGL : public QOpenGLWidget,
+		     public ViewFinder,
+		     protected QOpenGLFunctions
+{
+	Q_OBJECT
+
+public:
+	ViewFinderGL(QWidget *parent = nullptr);
+	~ViewFinderGL();
+
+	const QList<libcamera::PixelFormat> &nativeFormats() const override;
+
+	int setFormat(const libcamera::PixelFormat &format, const QSize &size) override;
+	void render(libcamera::FrameBuffer *buffer, MappedBuffer *map) override;
+	void stop() override;
+
+	QImage getCurrentImage() override;
+
+Q_SIGNALS:
+	void renderComplete(libcamera::FrameBuffer *buffer);
+
+protected:
+	void initializeGL() override;
+	void paintGL() override;
+	void resizeGL(int w, int h) override;
+	QSize sizeHint() const override;
+
+private:
+	bool selectFormat(const libcamera::PixelFormat &format);
+
+	void configureTexture(unsigned int id);
+	bool createFragmentShader();
+	bool createVertexShader();
+	void removeShader();
+	void doRender();
+
+	/* Captured image size, format and buffer */
+	libcamera::FrameBuffer *buffer_;
+	libcamera::PixelFormat format_;
+	QSize size_;
+	unsigned char *yuvData_;
+
+	/* OpenGL components for rendering */
+	QOpenGLShader *fragmentShader_;
+	QOpenGLShader *vertexShader_;
+	QOpenGLShaderProgram shaderProgram_;
+
+	/* Vertex buffer */
+	QOpenGLBuffer vertexBuffer_;
+
+	/* Fragment and Vertex shader file name */
+	QString fragmentShaderSrc_;
+	QString vertexShaderSrc_;
+
+	/* YUV texture planars and parameters */
+	GLuint id_u_;
+	GLuint id_v_;
+	GLuint id_y_;
+	GLuint textureUniformU_;
+	GLuint textureUniformV_;
+	GLuint textureUniformY_;
+	QOpenGLTexture textureU_;
+	QOpenGLTexture textureV_;
+	QOpenGLTexture textureY_;
+	unsigned int horzSubSample_;
+	unsigned int vertSubSample_;
+
+	QMutex mutex_; /* Prevent concurrent access to image_ */
+};
+
+#endif /* __VIEWFINDER_GL_H__ */