/* SPDX-License-Identifier: BSD-2-Clause */ /* * Copyright (C) 2019, Raspberry Pi (Trading) Limited * * awb.hpp - AWB control algorithm */ #pragma once #include #include #include #include "../awb_algorithm.hpp" #include "../pwl.hpp" #include "../awb_status.h" namespace RPiController { // Control algorithm to perform AWB calculations. struct AwbMode { void Read(boost::property_tree::ptree const ¶ms); double ct_lo; // low CT value for search double ct_hi; // high CT value for search }; struct AwbPrior { void Read(boost::property_tree::ptree const ¶ms); double lux; // lux level Pwl prior; // maps CT to prior log likelihood for this lux level }; struct AwbConfig { AwbConfig() : default_mode(nullptr) {} void Read(boost::property_tree::ptree const ¶ms); // Only repeat the AWB calculation every "this many" frames uint16_t frame_period; // number of initial frames for which speed taken as 1.0 (maximum) uint16_t startup_frames; unsigned int convergence_frames; // approx number of frames to converge double speed; // IIR filter speed applied to algorithm results bool fast; // "fast" mode uses a 16x16 rather than 32x32 grid Pwl ct_r; // function maps CT to r (= R/G) Pwl ct_b; // function maps CT to b (= B/G) // table of illuminant priors at different lux levels std::vector priors; // AWB "modes" (determines the search range) std::map modes; AwbMode *default_mode; // mode used if no mode selected // minimum proportion of pixels counted within AWB region for it to be // "useful" double min_pixels; // minimum G value of those pixels, to be regarded a "useful" uint16_t min_G; // number of AWB regions that must be "useful" in order to do the AWB // calculation uint32_t min_regions; // clamp on colour error term (so as not to penalise non-grey excessively) double delta_limit; // step size control in coarse search double coarse_step; // how far to wander off CT curve towards "more purple" double transverse_pos; // how far to wander off CT curve towards "more green" double transverse_neg; // red sensitivity ratio (set to canonical sensor's R/G divided by this // sensor's R/G) double sensitivity_r; // blue sensitivity ratio (set to canonical sensor's B/G divided by this // sensor's B/G) double sensitivity_b; // The whitepoint (which we normally "aim" for) can be moved. double whitepoint_r; double whitepoint_b; bool bayes; // use Bayesian algorithm }; class Awb : public AwbAlgorithm { public: Awb(Controller *controller = NULL); ~Awb(); char const *Name() const override; void Initialise() override; void Read(boost::property_tree::ptree const ¶ms) override; // AWB handles "pausing" for itself. bool IsPaused() const override; void Pause() override; void Resume() override; unsigned int GetConvergenceFrames() const override; void SetMode(std::string const &name) override; void SetManualGains(double manual_r, double manual_b) override; void SwitchMode(CameraMode const &camera_mode, Metadata *metadata) override; void Prepare(Metadata *image_metadata) override; void Process(StatisticsPtr &stats, Metadata *image_metadata) override; struct RGB { RGB(double _R = 0, double _G = 0, double _B = 0) : R(_R), G(_G), B(_B) { } double R, G, B; RGB &operator+=(RGB const &other) { R += other.R, G += other.G, B += other.B; return *this; } }; private: bool isAutoEnabled() const; // configuration is read-only, and available to both threads AwbConfig config_; std::thread async_thread_; void asyncFunc(); // asynchronous thread function std::mutex mutex_; // condvar for async thread to wait on std::condition_variable async_signal_; // condvar for synchronous thread to wait on std::condition_variable sync_signal_; // for sync thread to check if async thread finished (requires mutex) bool async_finished_; // for async thread to check if it's been told to run (requires mutex) bool async_start_; // for async thread to check if it's been told to quit (requires mutex) bool async_abort_; // The following are only for the synchronous thread to use: // for sync thread to note its has asked async thread to run bool async_started_; // counts up to frame_period before restarting the async thread int frame_phase_; int frame_count_; // counts up to startup_frames AwbStatus sync_results_; AwbStatus prev_sync_results_; std::string mode_name_; // The following are for the asynchronous thread to use, though the main // thread can set/reset them if the async thread is known to be idle: void restartAsync(StatisticsPtr &stats, double lux); // copy out the results from the async thread so that it can be restarted void fetchAsyncResults(); StatisticsPtr statistics_; AwbMode *mode_; double lux_; AwbStatus async_results_; void doAwb(); void awbBayes(); void awbGrey(); void prepareStats(); double computeDelta2Sum(double gain_r, double gain_b); Pwl interpolatePrior(); double coarseSearch(Pwl const &prior); void fineSearch(double &t, double &r, double &b, Pwl const &prior); std::vector zones_; std::vector points_; // manual r setting double manual_r_; // manual b setting double manual_b_; bool first_switch_mode_; // is this the first call to SwitchMode? }; static inline Awb::RGB operator+(Awb::RGB const &a, Awb::RGB const &b) { return Awb::RGB(a.R + b.R, a.G + b.G, a.B + b.B); } static inline Awb::RGB operator-(Awb::RGB const &a, Awb::RGB const &b) { return Awb::RGB(a.R - b.R, a.G - b.G, a.B - b.B); } static inline Awb::RGB operator*(double d, Awb::RGB const &rgb) { return Awb::RGB(d * rgb.R, d * rgb.G, d * rgb.B); } static inline Awb::RGB operator*(Awb::RGB const &rgb, double d) { return d * rgb; } } // namespace RPiController 'n69' href='#n69'>69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329

/* SPDX-License-Identifier: LGPL-2.1-or-later */
/*
 * Copyright (C) 2021-2022, Ideas On Board
 *
 * awb.cpp - AWB control algorithm
 */

#include "awb.h"

#include <algorithm>
#include <cmath>
#include <iomanip>

#include <libcamera/base/log.h>

#include <libcamera/control_ids.h>
#include <libcamera/ipa/core_ipa_interface.h>

/**
 * \file awb.h
 */

namespace libcamera {

namespace ipa::rkisp1::algorithms {

/**
 * \class Awb
 * \brief A Grey world white balance correction algorithm
 */

LOG_DEFINE_CATEGORY(RkISP1Awb)

/* Minimum mean value below which AWB can't operate. */
constexpr double kMeanMinThreshold = 2.0;

Awb::Awb()
	: rgbMode_(false)
{
}

/**
 * \copydoc libcamera::ipa::Algorithm::configure
 */
int Awb::configure(IPAContext &context,
		   const IPACameraSensorInfo &configInfo)
{
	context.activeState.awb.gains.manual.red = 1.0;
	context.activeState.awb.gains.manual.blue = 1.0;
	context.activeState.awb.gains.manual.green = 1.0;
	context.activeState.awb.gains.automatic.red = 1.0;
	context.activeState.awb.gains.automatic.blue = 1.0;
	context.activeState.awb.gains.automatic.green = 1.0;
	context.activeState.awb.autoEnabled = true;

	/*
	 * Define the measurement window for AWB as a centered rectangle
	 * covering 3/4 of the image width and height.
	 */
	context.configuration.awb.measureWindow.h_offs = configInfo.outputSize.width / 8;
	context.configuration.awb.measureWindow.v_offs = configInfo.outputSize.height / 8;
	context.configuration.awb.measureWindow.h_size = 3 * configInfo.outputSize.width / 4;
	context.configuration.awb.measureWindow.v_size = 3 * configInfo.outputSize.height / 4;

	context.configuration.awb.enabled = true;

	return 0;
}

/**
 * \copydoc libcamera::ipa::Algorithm::queueRequest
 */
void Awb::queueRequest(IPAContext &context,
		       [[maybe_unused]] const uint32_t frame,
		       IPAFrameContext &frameContext,
		       const ControlList &controls)
{
	auto &awb = context.activeState.awb;

	const auto &awbEnable = controls.get(controls::AwbEnable);
	if (awbEnable && *awbEnable != awb.autoEnabled) {
		awb.autoEnabled = *awbEnable;

		LOG(RkISP1Awb, Debug)
			<< (*awbEnable ? "Enabling" : "Disabling") << " AWB";
	}

	const auto &colourGains = controls.get(controls::ColourGains);
	if (colourGains && !awb.autoEnabled) {
		awb.gains.manual.red = (*colourGains)[0];
		awb.gains.manual.blue = (*colourGains)[1];

		LOG(RkISP1Awb, Debug)
			<< "Set colour gains to red: " << awb.gains.manual.red
			<< ", blue: " << awb.gains.manual.blue;
	}

	frameContext.awb.autoEnabled = awb.autoEnabled;

	if (!awb.autoEnabled) {
		frameContext.awb.gains.red = awb.gains.manual.red;
		frameContext.awb.gains.green = 1.0;
		frameContext.awb.gains.blue = awb.gains.manual.blue;
	}
}

/**
 * \copydoc libcamera::ipa::Algorithm::prepare
 */
void Awb::prepare(IPAContext &context, const uint32_t frame,
		  IPAFrameContext &frameContext, rkisp1_params_cfg *params)
{
	/*
	 * This is the latest time we can read the active state. This is the
	 * most up-to-date automatic values we can read.
	 */
	if (frameContext.awb.autoEnabled) {
		frameContext.awb.gains.red = context.activeState.awb.gains.automatic.red;
		frameContext.awb.gains.green = context.activeState.awb.gains.automatic.green;
		frameContext.awb.gains.blue = context.activeState.awb.gains.automatic.blue;
	}

	params->others.awb_gain_config.gain_green_b = 256 * frameContext.awb.gains.green;
	params->others.awb_gain_config.gain_blue = 256 * frameContext.awb.gains.blue;
	params->others.awb_gain_config.gain_red = 256 * frameContext.awb.gains.red;
	params->others.awb_gain_config.gain_green_r = 256 * frameContext.awb.gains.green;

	/* Update the gains. */
	params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;

	/* If we have already set the AWB measurement parameters, return. */
	if (frame > 0)
		return;

	rkisp1_cif_isp_awb_meas_config &awb_config = params->meas.awb_meas_config;

	/* Configure the measure window for AWB. */
	awb_config.awb_wnd = context.configuration.awb.measureWindow;

	/* Number of frames to use to estimate the means (0 means 1 frame). */
	awb_config.frames = 0;

	/* Select RGB or YCbCr means measurement. */
	if (rgbMode_) {
		awb_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_RGB;

		/*
		 * For RGB-based measurements, pixels are selected with maximum
		 * red, green and blue thresholds that are set in the
		 * awb_ref_cr, awb_min_y and awb_ref_cb respectively. The other
		 * values are not used, set them to 0.
		 */
		awb_config.awb_ref_cr = 250;
		awb_config.min_y = 250;
		awb_config.awb_ref_cb = 250;

		awb_config.max_y = 0;
		awb_config.min_c = 0;
		awb_config.max_csum = 0;
	} else {
		awb_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR;

		/* Set the reference Cr and Cb (AWB target) to white. */
		awb_config.awb_ref_cb = 128;
		awb_config.awb_ref_cr = 128;

		/*
		 * Filter out pixels based on luminance and chrominance values.
		 * The acceptable luma values are specified as a [16, 250]
		 * range, while the acceptable chroma values are specified with
		 * a minimum of 16 and a maximum Cb+Cr sum of 250.
		 */
		awb_config.min_y = 16;
		awb_config.max_y = 250;
		awb_config.min_c = 16;
		awb_config.max_csum = 250;
	}

	/* Enable the AWB gains. */
	params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;
	params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN;

	/* Update the AWB measurement parameters and enable the AWB module. */
	params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB;
	params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB;
	params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB;
}

uint32_t Awb::estimateCCT(double red, double green, double blue)
{
	/* Convert the RGB values to CIE tristimulus values (XYZ) */
	double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue);
	double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue);
	double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue);

	/* Calculate the normalized chromaticity values */
	double x = X / (X + Y + Z);
	double y = Y / (X + Y + Z);

	/* Calculate CCT */
	double n = (x - 0.3320) / (0.1858 - y);
	return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33;
}

/**
 * \copydoc libcamera::ipa::Algorithm::process
 */
void Awb::process(IPAContext &context,
		  [[maybe_unused]] const uint32_t frame,
		  IPAFrameContext &frameContext,
		  const rkisp1_stat_buffer *stats,
		  ControlList &metadata)
{
	const rkisp1_cif_isp_stat *params = &stats->params;
	const rkisp1_cif_isp_awb_stat *awb = &params->awb;
	IPAActiveState &activeState = context.activeState;
	double greenMean;
	double redMean;
	double blueMean;

	if (rgbMode_) {
		greenMean = awb->awb_mean[0].mean_y_or_g;
		redMean = awb->awb_mean[0].mean_cr_or_r;
		blueMean = awb->awb_mean[0].mean_cb_or_b;
	} else {
		/* Get the YCbCr mean values */
		double yMean = awb->awb_mean[0].mean_y_or_g;
		double cbMean = awb->awb_mean[0].mean_cb_or_b;
		double crMean = awb->awb_mean[0].mean_cr_or_r;

		/*
		 * Convert from YCbCr to RGB.
		 * The hardware uses the following formulas:
		 * Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B
		 * Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B
		 * Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B
		 *
		 * The inverse matrix is thus:
		 * [[1,1636, -0,0623,  1,6008]
		 *  [1,1636, -0,4045, -0,7949]
		 *  [1,1636,  1,9912, -0,0250]]
		 */
		yMean -= 16;
		cbMean -= 128;
		crMean -= 128;
		redMean = 1.1636 * yMean - 0.0623 * cbMean + 1.6008 * crMean;
		greenMean = 1.1636 * yMean - 0.4045 * cbMean - 0.7949 * crMean;
		blueMean = 1.1636 * yMean + 1.9912 * cbMean - 0.0250 * crMean;

		/*
		 * Due to hardware rounding errors in the YCbCr means, the
		 * calculated RGB means may be negative. This would lead to
		 * negative gains, messing up calculation. Prevent this by
		 * clamping the means to positive values.
		 */
		redMean = std::max(redMean, 0.0);
		greenMean = std::max(greenMean, 0.0);
		blueMean = std::max(blueMean, 0.0);
	}

	/*
	 * The ISP computes the AWB means after applying the colour gains,
	 * divide by the gains that were used to get the raw means from the
	 * sensor.
	 */
	redMean /= frameContext.awb.gains.red;
	greenMean /= frameContext.awb.gains.green;
	blueMean /= frameContext.awb.gains.blue;

	/*
	 * If the means are too small we don't have enough information to
	 * meaningfully calculate gains. Freeze the algorithm in that case.
	 */
	if (redMean < kMeanMinThreshold && greenMean < kMeanMinThreshold &&
	    blueMean < kMeanMinThreshold) {
		frameContext.awb.temperatureK = activeState.awb.temperatureK;
		return;
	}

	activeState.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean);

	/*
	 * Estimate the red and blue gains to apply in a grey world. The green
	 * gain is hardcoded to 1.0. Avoid divisions by zero by clamping the
	 * divisor to a minimum value of 1.0.
	 */
	double redGain = greenMean / std::max(redMean, 1.0);
	double blueGain = greenMean / std::max(blueMean, 1.0);

	/*
	 * Clamp the gain values to the hardware, which expresses gains as Q2.8
	 * unsigned integer values. Set the minimum just above zero to avoid
	 * divisions by zero when computing the raw means in subsequent
	 * iterations.
	 */
	redGain = std::clamp(redGain, 1.0 / 256, 1023.0 / 256);
	blueGain = std::clamp(blueGain, 1.0 / 256, 1023.0 / 256);

	/* Filter the values to avoid oscillations. */
	double speed = 0.2;
	redGain = speed * redGain + (1 - speed) * activeState.awb.gains.automatic.red;
	blueGain = speed * blueGain + (1 - speed) * activeState.awb.gains.automatic.blue;

	activeState.awb.gains.automatic.red = redGain;
	activeState.awb.gains.automatic.blue = blueGain;
	activeState.awb.gains.automatic.green = 1.0;

	frameContext.awb.temperatureK = activeState.awb.temperatureK;

	metadata.set(controls::AwbEnable, frameContext.awb.autoEnabled);
	metadata.set(controls::ColourGains, {
			static_cast<float>(frameContext.awb.gains.red),
			static_cast<float>(frameContext.awb.gains.blue)
		});
	metadata.set(controls::ColourTemperature, frameContext.awb.temperatureK);

	LOG(RkISP1Awb, Debug) << std::showpoint
		<< "Means [" << redMean << ", " << greenMean << ", " << blueMean
		<< "], gains [" << activeState.awb.gains.automatic.red << ", "
		<< activeState.awb.gains.automatic.green << ", "
		<< activeState.awb.gains.automatic.blue << "], temp "
		<< frameContext.awb.temperatureK << "K";
}

REGISTER_IPA_ALGORITHM(Awb, "Awb")

} /* namespace ipa::rkisp1::algorithms */

} /* namespace libcamera */