From 251f0534b74bcb46c777aa0df34b1b4142b664f5 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sun, 28 Aug 2022 03:40:25 +0300 Subject: qcam: viewfinder_gl: Take color space into account for YUV rendering Update the YUV shaders and the viewfinder_gl to correctly take the Y'CbCr encoding and the quantization range into account when rendering YUV formats to RGB. Support for the primaries and transfer function will be added in a subsequent step. Signed-off-by: Laurent Pinchart Reviewed-by: Umang Jain Reviewed-by: Kunal Agarwal --- src/qcam/assets/shader/YUV_2_planes.frag | 29 ++++++------ src/qcam/assets/shader/YUV_3_planes.frag | 27 ++++++----- src/qcam/assets/shader/YUV_packed.frag | 17 +++---- src/qcam/viewfinder_gl.cpp | 79 ++++++++++++++++++++++++++++++-- src/qcam/viewfinder_gl.h | 2 + 5 files changed, 118 insertions(+), 36 deletions(-) (limited to 'src/qcam') diff --git a/src/qcam/assets/shader/YUV_2_planes.frag b/src/qcam/assets/shader/YUV_2_planes.frag index 254463c0..da8dbcc5 100644 --- a/src/qcam/assets/shader/YUV_2_planes.frag +++ b/src/qcam/assets/shader/YUV_2_planes.frag @@ -13,27 +13,30 @@ varying vec2 textureOut; uniform sampler2D tex_y; uniform sampler2D tex_u; +const mat3 yuv2rgb_matrix = mat3( + YUV2RGB_MATRIX +); + +const vec3 yuv2rgb_offset = vec3( + YUV2RGB_Y_OFFSET / 255.0, 128.0 / 255.0, 128.0 / 255.0 +); + void main(void) { vec3 yuv; - vec3 rgb; - mat3 yuv2rgb_bt601_mat = mat3( - vec3(1.164, 1.164, 1.164), - vec3(0.000, -0.392, 2.017), - vec3(1.596, -0.813, 0.000) - ); - - yuv.x = texture2D(tex_y, textureOut).r - 0.063; + + yuv.x = texture2D(tex_y, textureOut).r; #if defined(YUV_PATTERN_UV) - yuv.y = texture2D(tex_u, textureOut).r - 0.500; - yuv.z = texture2D(tex_u, textureOut).a - 0.500; + yuv.y = texture2D(tex_u, textureOut).r; + yuv.z = texture2D(tex_u, textureOut).a; #elif defined(YUV_PATTERN_VU) - yuv.y = texture2D(tex_u, textureOut).a - 0.500; - yuv.z = texture2D(tex_u, textureOut).r - 0.500; + yuv.y = texture2D(tex_u, textureOut).a; + yuv.z = texture2D(tex_u, textureOut).r; #else #error Invalid pattern #endif - rgb = yuv2rgb_bt601_mat * yuv; + vec3 rgb = yuv2rgb_matrix * (vec3(y, uv) - yuv2rgb_offset); + gl_FragColor = vec4(rgb, 1.0); } diff --git a/src/qcam/assets/shader/YUV_3_planes.frag b/src/qcam/assets/shader/YUV_3_planes.frag index 2be74b5d..e754129d 100644 --- a/src/qcam/assets/shader/YUV_3_planes.frag +++ b/src/qcam/assets/shader/YUV_3_planes.frag @@ -14,20 +14,23 @@ uniform sampler2D tex_y; uniform sampler2D tex_u; uniform sampler2D tex_v; +const mat3 yuv2rgb_matrix = mat3( + YUV2RGB_MATRIX +); + +const vec3 yuv2rgb_offset = vec3( + YUV2RGB_Y_OFFSET / 255.0, 128.0 / 255.0, 128.0 / 255.0 +); + void main(void) { vec3 yuv; - vec3 rgb; - mat3 yuv2rgb_bt601_mat = mat3( - vec3(1.164, 1.164, 1.164), - vec3(0.000, -0.392, 2.017), - vec3(1.596, -0.813, 0.000) - ); - - yuv.x = texture2D(tex_y, textureOut).r - 0.063; - yuv.y = texture2D(tex_u, textureOut).r - 0.500; - yuv.z = texture2D(tex_v, textureOut).r - 0.500; - - rgb = yuv2rgb_bt601_mat * yuv; + + yuv.x = texture2D(tex_y, textureOut).r; + yuv.y = texture2D(tex_u, textureOut).r; + yuv.z = texture2D(tex_v, textureOut).r; + + vec3 rgb = yuv2rgb_matrix * (vec3(y, uv) - yuv2rgb_offset); + gl_FragColor = vec4(rgb, 1.0); } diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag index d6efd4ce..b9ef9d41 100644 --- a/src/qcam/assets/shader/YUV_packed.frag +++ b/src/qcam/assets/shader/YUV_packed.frag @@ -14,15 +14,16 @@ varying vec2 textureOut; uniform sampler2D tex_y; uniform vec2 tex_step; +const mat3 yuv2rgb_matrix = mat3( + YUV2RGB_MATRIX +); + +const vec3 yuv2rgb_offset = vec3( + YUV2RGB_Y_OFFSET / 255.0, 128.0 / 255.0, 128.0 / 255.0 +); + void main(void) { - mat3 yuv2rgb_bt601_mat = mat3( - vec3(1.164, 1.164, 1.164), - vec3(0.000, -0.392, 2.017), - vec3(1.596, -0.813, 0.000) - ); - vec3 yuv2rgb_bt601_offset = vec3(0.063, 0.500, 0.500); - /* * The sampler won't interpolate the texture correctly along the X axis, * as each RGBA pixel effectively stores two pixels. We thus need to @@ -76,7 +77,7 @@ void main(void) float y = mix(y_left, y_right, step(0.5, f_x)); - vec3 rgb = yuv2rgb_bt601_mat * (vec3(y, uv) - yuv2rgb_bt601_offset); + vec3 rgb = yuv2rgb_matrix * (vec3(y, uv) - yuv2rgb_offset); gl_FragColor = vec4(rgb, 1.0); } diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp index ec295b6d..e2aa2470 100644 --- a/src/qcam/viewfinder_gl.cpp +++ b/src/qcam/viewfinder_gl.cpp @@ -7,9 +7,12 @@ #include "viewfinder_gl.h" +#include + #include #include #include +#include #include @@ -56,7 +59,8 @@ static const QList supportedFormats{ }; ViewFinderGL::ViewFinderGL(QWidget *parent) - : QOpenGLWidget(parent), buffer_(nullptr), image_(nullptr), + : QOpenGLWidget(parent), buffer_(nullptr), + colorSpace_(libcamera::ColorSpace::Raw), image_(nullptr), vertexBuffer_(QOpenGLBuffer::VertexBuffer) { } @@ -72,10 +76,10 @@ const QList &ViewFinderGL::nativeFormats() const } int ViewFinderGL::setFormat(const libcamera::PixelFormat &format, const QSize &size, - [[maybe_unused]] const libcamera::ColorSpace &colorSpace, + const libcamera::ColorSpace &colorSpace, unsigned int stride) { - if (format != format_) { + if (format != format_ || colorSpace != colorSpace_) { /* * If the fragment already exists, remove it and create a new * one for the new format. @@ -89,7 +93,10 @@ int ViewFinderGL::setFormat(const libcamera::PixelFormat &format, const QSize &s if (!selectFormat(format)) return -1; + selectColorSpace(colorSpace); + format_ = format; + colorSpace_ = colorSpace; } size_ = size; @@ -318,6 +325,72 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) return ret; } +void ViewFinderGL::selectColorSpace(const libcamera::ColorSpace &colorSpace) +{ + std::array yuv2rgb; + + /* OpenGL stores arrays in column-major order. */ + switch (colorSpace.ycbcrEncoding) { + case libcamera::ColorSpace::YcbcrEncoding::None: + yuv2rgb = { + 1.0000, 0.0000, 0.0000, + 0.0000, 1.0000, 0.0000, + 0.0000, 0.0000, 1.0000, + }; + break; + + case libcamera::ColorSpace::YcbcrEncoding::Rec601: + yuv2rgb = { + 1.0000, 1.0000, 1.0000, + 0.0000, -0.3441, 1.7720, + 1.4020, -0.7141, 0.0000, + }; + break; + + case libcamera::ColorSpace::YcbcrEncoding::Rec709: + yuv2rgb = { + 1.0000, 1.0000, 1.0000, + 0.0000, -0.1873, 1.8856, + 1.5748, -0.4681, 0.0000, + }; + break; + + case libcamera::ColorSpace::YcbcrEncoding::Rec2020: + yuv2rgb = { + 1.0000, 1.0000, 1.0000, + 0.0000, -0.1646, 1.8814, + 1.4746, -0.5714, 0.0000, + }; + break; + } + + double offset; + + switch (colorSpace.range) { + case libcamera::ColorSpace::Range::Full: + offset = 0.0; + break; + + case libcamera::ColorSpace::Range::Limited: + offset = 16.0; + + for (unsigned int i = 0; i < 3; ++i) + yuv2rgb[i] *= 255.0 / 219.0; + for (unsigned int i = 4; i < 9; ++i) + yuv2rgb[i] *= 255.0 / 224.0; + break; + } + + QStringList matrix; + + for (double coeff : yuv2rgb) + matrix.append(QString::number(coeff, 'f')); + + fragmentShaderDefines_.append("#define YUV2RGB_MATRIX " + matrix.join(", ")); + fragmentShaderDefines_.append(QString("#define YUV2RGB_Y_OFFSET %1") + .arg(offset, 0, 'f', 1)); +} + bool ViewFinderGL::createVertexShader() { /* Create Vertex Shader */ diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h index 798830a3..68c2912d 100644 --- a/src/qcam/viewfinder_gl.h +++ b/src/qcam/viewfinder_gl.h @@ -57,6 +57,7 @@ protected: private: bool selectFormat(const libcamera::PixelFormat &format); + void selectColorSpace(const libcamera::ColorSpace &colorSpace); void configureTexture(QOpenGLTexture &texture); bool createFragmentShader(); @@ -67,6 +68,7 @@ private: /* Captured image size, format and buffer */ libcamera::FrameBuffer *buffer_; libcamera::PixelFormat format_; + libcamera::ColorSpace colorSpace_; QSize size_; unsigned int stride_; Image *image_; -- cgit v1.2.1