qcam: viewfinder_gl: Add shader to render packed RAW10 formats

The shader supports all 4 packed RAW10 variants.
Simple bi-linear Bayer interpolation of nearest pixels is implemented.
The 2 LS bits of the 10-bit colour values are dropped as the RGBA
format we convert into has only 8 bits per colour.

The texture coordinates passed to the fragment shader are adjusted
to point to the nearest pixel in the image. This prevents artifacts
when the image is scaled from the frame resolution to the window size.

Signed-off-by: Andrey Konovalov <andrey.konovalov@linaro.org>
Reviewed-by: Paul Elder <paul.elder@ideasonboard.com>
Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>

4 files changed, 271 insertions, 2 deletions

diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag
new file mode 100644
index 00000000..d36db8a2
--- /dev/null
+++ b/src/qcam/assets/shader/bayer_1x_packed.frag
@@ -0,0 +1,185 @@
+/* SPDX-License-Identifier: BSD-2-Clause */
+/*
+ * Based on the code from http://jgt.akpeters.com/papers/McGuire08/
+ *
+ * Efficient, High-Quality Bayer Demosaic Filtering on GPUs
+ *
+ * Morgan McGuire
+ *
+ * This paper appears in issue Volume 13, Number 4.
+ * ---------------------------------------------------------
+ * Copyright (c) 2008, Morgan McGuire. All rights reserved.
+ *
+ *
+ * Modified by Linaro Ltd for 10/12-bit packed vs 8-bit raw Bayer format,
+ * and for simpler demosaic algorithm.
+ * Copyright (C) 2020, Linaro
+ *
+ * bayer_1x_packed.frag - Fragment shader code for raw Bayer 10-bit and 12-bit
+ * packed formats
+ */
+
+#ifdef GL_ES
+precision mediump float;
+#endif
+
+varying vec2 textureOut;
+
+/* the texture size in pixels */
+uniform vec2 tex_size;
+uniform vec2 tex_step;
+uniform vec2 tex_bayer_first_red;
+
+uniform sampler2D tex_y;
+
+void main(void)
+{
+	vec3 rgb;
+
+	/*
+	 * center_bytes holds the coordinates of the MS byte of the pixel
+	 * being sampled on the [0, stride-1/height-1] range.
+	 * center_pixel holds the coordinates of the pixel being sampled
+	 * on the [0, width/height-1] range.
+	 */
+	vec2 center_bytes;
+	vec2 center_pixel;
+
+	/*
+	 * x- and y-positions of the adjacent pixels on the [0, 1] range.
+	 */
+	vec2 xcoords;
+	vec2 ycoords;
+
+	/*
+	 * The coordinates passed to the shader in textureOut may point
+	 * to a place in between the pixels if the texture format doesn't
+	 * match the image format. In particular, MIPI packed raw Bayer
+	 * formats don't have a matching texture format.
+	 * In this case align the coordinates to the left nearest pixel
+	 * by hand.
+	 */
+	center_pixel = floor(textureOut * tex_size);
+	center_bytes.y = center_pixel.y;
+
+	/*
+	 * Add a small number (a few mantissa's LSBs) to avoid float
+	 * representation issues. Maybe paranoic.
+	 */
+	center_bytes.x = BPP_X * center_pixel.x + 0.02;
+
+	const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */;
+	const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */;
+
+	float fract_x = fract(center_bytes.x);
+
+	/*
+	 * The below floor() call ensures that center_bytes.x points
+	 * at one of the bytes representing the 8 higher bits of
+	 * the pixel value, not at the byte containing the LS bits
+	 * of the group of the pixels.
+	 */
+	center_bytes.x = floor(center_bytes.x);
+	center_bytes *= tex_step;
+
+	xcoords = center_bytes.x + vec2(-tex_step.x, tex_step.x);
+	ycoords = center_bytes.y + vec2(-tex_step.y, tex_step.y);
+
+	/*
+	 * If xcoords[0] points at the byte containing the LS bits
+	 * of the previous group of the pixels, move xcoords[0] one
+	 * byte back.
+	 */
+	xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0;
+
+	/*
+	 * If xcoords[1] points at the byte containing the LS bits
+	 * of the current group of the pixels, move xcoords[1] one
+	 * byte forward.
+	 */
+	xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0;
+
+	vec2 alternate = mod(center_pixel.xy + tex_bayer_first_red, 2.0);
+	bool even_col = alternate.x < 1.0;
+	bool even_row = alternate.y < 1.0;
+
+	/*
+	 * We need to sample the central pixel and the ones with offset
+	 * of -1 to +1 pixel in both X and Y directions. Let's name these
+	 * pixels as below, where C is the central pixel:
+	 *
+	 *   +----+----+----+----+
+	 *   | \ x|    |    |    |
+	 *   |y \ | -1 |  0 | +1 |
+	 *   +----+----+----+----+
+	 *   | +1 | D2 | A1 | D3 |
+	 *   +----+----+----+----+
+	 *   |  0 | B0 |  C | B1 |
+	 *   +----+----+----+----+
+	 *   | -1 | D0 | A0 | D1 |
+	 *   +----+----+----+----+
+	 *
+	 * In the below equations (0,-1).r means "r component of the texel
+	 * shifted by -tex_step.y from the center_bytes one" etc.
+	 *
+	 * In the even row / even column (EE) case the colour values are:
+	 *   R = C = (0,0).r,
+	 *   G = (A0 + A1 + B0 + B1) / 4.0 =
+	 *       ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0,
+	 *   B = (D0 + D1 + D2 + D3) / 4.0 =
+	 *       ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0
+	 *
+	 * For even row / odd column (EO):
+	 *   R = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0,
+	 *   G = C = (0,0).r,
+	 *   B = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0
+	 *
+	 * For odd row / even column (OE):
+	 *   R = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0,
+	 *   G = C = (0,0).r,
+	 *   B = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0
+	 *
+	 * For odd row / odd column (OO):
+	 *   R = (D0 + D1 + D2 + D3) / 4.0 =
+	 *       ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0,
+	 *   G = (A0 + A1 + B0 + B1) / 4.0 =
+	 *       ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0,
+	 *   B = C = (0,0).r
+	 */
+
+	/*
+	 * Fetch the values and precalculate the terms:
+	 *   patterns.x = (A0 + A1) / 2.0
+	 *   patterns.y = (B0 + B1) / 2.0
+	 *   patterns.z = (A0 + A1 + B0 + B1) / 4.0
+	 *   patterns.w = (D0 + D1 + D2 + D3) / 4.0
+	 */
+	#define fetch(x, y) texture2D(tex_y, vec2(x, y)).r
+
+	float C = texture2D(tex_y, center_bytes).r;
+	vec4 patterns = vec4(
+		fetch(center_bytes.x, ycoords[0]),	/* A0: (0,-1) */
+		fetch(xcoords[0], center_bytes.y),	/* B0: (-1,0) */
+		fetch(xcoords[0], ycoords[0]),		/* D0: (-1,-1) */
+		fetch(xcoords[1], ycoords[0]));		/* D1: (1,-1) */
+	vec4 temp = vec4(
+		fetch(center_bytes.x, ycoords[1]),	/* A1: (0,1) */
+		fetch(xcoords[1], center_bytes.y),	/* B1: (1,0) */
+		fetch(xcoords[1], ycoords[1]),		/* D3: (1,1) */
+		fetch(xcoords[0], ycoords[1]));		/* D2: (-1,1) */
+	patterns = (patterns + temp) * 0.5;
+		/* .x = (A0 + A1) / 2.0, .y = (B0 + B1) / 2.0 */
+		/* .z = (D0 + D3) / 2.0, .w = (D1 + D2) / 2.0 */
+	patterns.w = (patterns.z + patterns.w) * 0.5;
+	patterns.z = (patterns.x + patterns.y) * 0.5;
+
+	rgb = even_col ?
+		(even_row ?
+			vec3(C, patterns.zw) :
+			vec3(patterns.x, C, patterns.y)) :
+		(even_row ?
+			vec3(patterns.y, C, patterns.x) :
+			vec3(patterns.wz, C));
+
+	gl_FragColor = vec4(rgb, 1.0);
+}
diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc
index 8a8f9de1..d76d65c5 100644
--- a/src/qcam/assets/shader/shaders.qrc
+++ b/src/qcam/assets/shader/shaders.qrc
@@ -5,6 +5,7 @@
 	<file>YUV_2_planes.frag</file>
 	<file>YUV_3_planes.frag</file>
 	<file>YUV_packed.frag</file>
+	<file>bayer_1x_packed.frag</file>
 	<file>identity.vert</file>
 </qresource>
 </RCC>
diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp
index ff719418..ffbbc6c5 100644
--- a/src/qcam/viewfinder_gl.cpp
+++ b/src/qcam/viewfinder_gl.cpp
@@ -36,6 +36,11 @@ static const QList<libcamera::PixelFormat> supportedFormats{
 	libcamera::formats::RGBA8888,
 	libcamera::formats::BGR888,
 	libcamera::formats::RGB888,
+	/* Raw Bayer 10-bit packed */
+	libcamera::formats::SBGGR10_CSI2P,
+	libcamera::formats::SGBRG10_CSI2P,
+	libcamera::formats::SGRBG10_CSI2P,
+	libcamera::formats::SRGGB10_CSI2P,
 };
 
 ViewFinderGL::ViewFinderGL(QWidget *parent)
@@ -106,6 +111,10 @@ void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map)
 		renderComplete(buffer_);
 
 	data_ = static_cast<unsigned char *>(map->memory);
+	/*
+	 * \todo Get the stride from the buffer instead of computing it naively
+	 */
+	stride_ = buffer->metadata().planes[0].bytesused / size_.height();
 	update();
 	buffer_ = buffer;
 }
@@ -114,6 +123,9 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
 {
 	bool ret = true;
 
+	/* Set min/mag filters to GL_LINEAR by default. */
+	textureMinMagFilters_ = GL_LINEAR;
+
 	fragmentShaderDefines_.clear();
 
 	switch (format) {
@@ -203,6 +215,34 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
 		fragmentShaderDefines_.append("#define RGB_PATTERN bgr");
 		fragmentShaderFile_ = ":RGB.frag";
 		break;
+	case libcamera::formats::SBGGR10_CSI2P:
+		firstRed_.setX(1.0);
+		firstRed_.setY(1.0);
+		fragmentShaderDefines_.append("#define BPP_X 1.25");
+		fragmentShaderFile_ = ":bayer_1x_packed.frag";
+		textureMinMagFilters_ = GL_NEAREST;
+		break;
+	case libcamera::formats::SGBRG10_CSI2P:
+		firstRed_.setX(0.0);
+		firstRed_.setY(1.0);
+		fragmentShaderDefines_.append("#define BPP_X 1.25");
+		fragmentShaderFile_ = ":bayer_1x_packed.frag";
+		textureMinMagFilters_ = GL_NEAREST;
+		break;
+	case libcamera::formats::SGRBG10_CSI2P:
+		firstRed_.setX(1.0);
+		firstRed_.setY(0.0);
+		fragmentShaderDefines_.append("#define BPP_X 1.25");
+		fragmentShaderFile_ = ":bayer_1x_packed.frag";
+		textureMinMagFilters_ = GL_NEAREST;
+		break;
+	case libcamera::formats::SRGGB10_CSI2P:
+		firstRed_.setX(0.0);
+		firstRed_.setY(0.0);
+		fragmentShaderDefines_.append("#define BPP_X 1.25");
+		fragmentShaderFile_ = ":bayer_1x_packed.frag";
+		textureMinMagFilters_ = GL_NEAREST;
+		break;
 	default:
 		ret = false;
 		qWarning() << "[ViewFinderGL]:"
@@ -290,6 +330,8 @@ bool ViewFinderGL::createFragmentShader()
 	textureUniformU_ = shaderProgram_.uniformLocation("tex_u");
 	textureUniformV_ = shaderProgram_.uniformLocation("tex_v");
 	textureUniformStep_ = shaderProgram_.uniformLocation("tex_step");
+	textureUniformSize_ = shaderProgram_.uniformLocation("tex_size");
+	textureUniformBayerFirstRed_ = shaderProgram_.uniformLocation("tex_bayer_first_red");
 
 	/* Create the textures. */
 	for (std::unique_ptr<QOpenGLTexture> &texture : textures_) {
@@ -306,8 +348,10 @@ bool ViewFinderGL::createFragmentShader()
 void ViewFinderGL::configureTexture(QOpenGLTexture &texture)
 {
 	glBindTexture(GL_TEXTURE_2D, texture.textureId());
-	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
-	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
+			textureMinMagFilters_);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
+			textureMinMagFilters_);
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 }
@@ -547,6 +591,36 @@ void ViewFinderGL::doRender()
 		shaderProgram_.setUniformValue(textureUniformY_, 0);
 		break;
 
+	case libcamera::formats::SBGGR10_CSI2P:
+	case libcamera::formats::SGBRG10_CSI2P:
+	case libcamera::formats::SGRBG10_CSI2P:
+	case libcamera::formats::SRGGB10_CSI2P:
+		/*
+		 * Packed raw Bayer 10-bit formats are stored in GL_RED texture.
+		 * The texture width is equal to the stride.
+		 */
+		glActiveTexture(GL_TEXTURE0);
+		configureTexture(*textures_[0]);
+		glTexImage2D(GL_TEXTURE_2D,
+			     0,
+			     GL_RED,
+			     stride_,
+			     size_.height(),
+			     0,
+			     GL_RED,
+			     GL_UNSIGNED_BYTE,
+			     data_);
+		shaderProgram_.setUniformValue(textureUniformY_, 0);
+		shaderProgram_.setUniformValue(textureUniformBayerFirstRed_,
+					       firstRed_);
+		shaderProgram_.setUniformValue(textureUniformSize_,
+					       size_.width(), /* in pixels */
+					       size_.height());
+		shaderProgram_.setUniformValue(textureUniformStep_,
+					       1.0f / (stride_ - 1),
+					       1.0f / (size_.height() - 1));
+		break;
+
 	default:
 		break;
 	};
diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h
index 1b1faa91..508155b1 100644
--- a/src/qcam/viewfinder_gl.h
+++ b/src/qcam/viewfinder_gl.h
@@ -66,6 +66,7 @@ private:
 	libcamera::FrameBuffer *buffer_;
 	libcamera::PixelFormat format_;
 	QSize size_;
+	unsigned int stride_;
 	unsigned char *data_;
 
 	/* Shaders */
@@ -81,6 +82,9 @@ private:
 	/* Textures */
 	std::array<std::unique_ptr<QOpenGLTexture>, 3> textures_;
 
+	/* Common texture parameters */
+	GLuint textureMinMagFilters_;
+
 	/* YUV texture parameters */
 	GLuint textureUniformU_;
 	GLuint textureUniformV_;
@@ -89,6 +93,11 @@ private:
 	unsigned int horzSubSample_;
 	unsigned int vertSubSample_;
 
+	/* Raw Bayer texture parameters */
+	GLuint textureUniformSize_;
+	GLuint textureUniformBayerFirstRed_;
+	QPointF firstRed_;
+
 	QMutex mutex_; /* Prevent concurrent access to image_ */
 };