diff options
Diffstat (limited to 'src/ipa')
68 files changed, 3607 insertions, 1129 deletions
diff --git a/src/ipa/ipu3/algorithms/awb.h b/src/ipa/ipu3/algorithms/awb.h index 1916990a..dbf69c90 100644 --- a/src/ipa/ipu3/algorithms/awb.h +++ b/src/ipa/ipu3/algorithms/awb.h @@ -13,7 +13,7 @@ #include <libcamera/geometry.h> -#include "libipa/vector.h" +#include "libcamera/internal/vector.h" #include "algorithm.h" diff --git a/src/ipa/libipa/camera_sensor_helper.cpp b/src/ipa/libipa/camera_sensor_helper.cpp index a76e5e75..7c66cd57 100644 --- a/src/ipa/libipa/camera_sensor_helper.cpp +++ b/src/ipa/libipa/camera_sensor_helper.cpp @@ -87,21 +87,16 @@ namespace ipa { */ uint32_t CameraSensorHelper::gainCode(double gain) const { - const AnalogueGainConstants &k = gainConstants_; + if (auto *l = std::get_if<AnalogueGainLinear>(&gain_)) { + ASSERT(l->m0 == 0 || l->m1 == 0); - switch (gainType_) { - case AnalogueGainLinear: - ASSERT(k.linear.m0 == 0 || k.linear.m1 == 0); + return (l->c0 - l->c1 * gain) / + (l->m1 * gain - l->m0); + } else if (auto *e = std::get_if<AnalogueGainExp>(&gain_)) { + ASSERT(e->a != 0 && e->m != 0); - return (k.linear.c0 - k.linear.c1 * gain) / - (k.linear.m1 * gain - k.linear.m0); - - case AnalogueGainExponential: - ASSERT(k.exp.a != 0 && k.exp.m != 0); - - return std::log2(gain / k.exp.a) / k.exp.m; - - default: + return std::log2(gain / e->a) / e->m; + } else { ASSERT(false); return 0; } @@ -119,38 +114,26 @@ uint32_t CameraSensorHelper::gainCode(double gain) const */ double CameraSensorHelper::gain(uint32_t gainCode) const { - const AnalogueGainConstants &k = gainConstants_; double gain = static_cast<double>(gainCode); - switch (gainType_) { - case AnalogueGainLinear: - ASSERT(k.linear.m0 == 0 || k.linear.m1 == 0); - - return (k.linear.m0 * gain + k.linear.c0) / - (k.linear.m1 * gain + k.linear.c1); + if (auto *l = std::get_if<AnalogueGainLinear>(&gain_)) { + ASSERT(l->m0 == 0 || l->m1 == 0); - case AnalogueGainExponential: - ASSERT(k.exp.a != 0 && k.exp.m != 0); + return (l->m0 * gain + l->c0) / + (l->m1 * gain + l->c1); + } else if (auto *e = std::get_if<AnalogueGainExp>(&gain_)) { + ASSERT(e->a != 0 && e->m != 0); - return k.exp.a * std::exp2(k.exp.m * gain); - - default: + return e->a * std::exp2(e->m * gain); + } else { ASSERT(false); return 0.0; } } /** - * \enum CameraSensorHelper::AnalogueGainType - * \brief The gain calculation modes as defined by the MIPI CCS - * - * Describes the image sensor analogue gain capabilities. - * Two modes are possible, depending on the sensor: Linear and Exponential. - */ - -/** - * \var CameraSensorHelper::AnalogueGainLinear - * \brief Gain is computed using linear gain estimation + * \struct CameraSensorHelper::AnalogueGainLinear + * \brief Analogue gain constants for the linear gain model * * The relationship between the integer gain parameter and the resulting gain * multiplier is given by the following equation: @@ -165,11 +148,27 @@ double CameraSensorHelper::gain(uint32_t gainCode) const * The full Gain equation therefore reduces to either: * * \f$gain=\frac{c0}{m1x+c1}\f$ or \f$\frac{m0x+c0}{c1}\f$ + * + * \var CameraSensorHelper::AnalogueGainLinear::m0 + * \brief Constant used in the linear gain coding/decoding + * + * \note Either m0 or m1 shall be zero. + * + * \var CameraSensorHelper::AnalogueGainLinear::c0 + * \brief Constant used in the linear gain coding/decoding + * + * \var CameraSensorHelper::AnalogueGainLinear::m1 + * \brief Constant used in the linear gain coding/decoding + * + * \note Either m0 or m1 shall be zero. + * + * \var CameraSensorHelper::AnalogueGainLinear::c1 + * \brief Constant used in the linear gain coding/decoding */ /** - * \var CameraSensorHelper::AnalogueGainExponential - * \brief Gain is expressed using an exponential model + * \struct CameraSensorHelper::AnalogueGainExp + * \brief Analogue gain constants for the exponential gain model * * The relationship between the integer gain parameter and the resulting gain * multiplier is given by the following equation: @@ -185,67 +184,22 @@ double CameraSensorHelper::gain(uint32_t gainCode) const * * When the gain is expressed in dB, 'a' is equal to 1 and 'm' to * \f$log_{2}{10^{\frac{1}{20}}}\f$. - */ - -/** - * \struct CameraSensorHelper::AnalogueGainLinearConstants - * \brief Analogue gain constants for the linear gain model * - * \var CameraSensorHelper::AnalogueGainLinearConstants::m0 - * \brief Constant used in the linear gain coding/decoding - * - * \note Either m0 or m1 shall be zero. - * - * \var CameraSensorHelper::AnalogueGainLinearConstants::c0 - * \brief Constant used in the linear gain coding/decoding - * - * \var CameraSensorHelper::AnalogueGainLinearConstants::m1 - * \brief Constant used in the linear gain coding/decoding - * - * \note Either m0 or m1 shall be zero. - * - * \var CameraSensorHelper::AnalogueGainLinearConstants::c1 - * \brief Constant used in the linear gain coding/decoding - */ - -/** - * \struct CameraSensorHelper::AnalogueGainExpConstants - * \brief Analogue gain constants for the exponential gain model - * - * \var CameraSensorHelper::AnalogueGainExpConstants::a + * \var CameraSensorHelper::AnalogueGainExp::a * \brief Constant used in the exponential gain coding/decoding * - * \var CameraSensorHelper::AnalogueGainExpConstants::m + * \var CameraSensorHelper::AnalogueGainExp::m * \brief Constant used in the exponential gain coding/decoding */ /** - * \struct CameraSensorHelper::AnalogueGainConstants - * \brief Analogue gain model constants - * - * This union stores the constants used to calculate the analogue gain. The - * CameraSensorHelper::gainType_ variable selects which union member is valid. - * - * \var CameraSensorHelper::AnalogueGainConstants::linear - * \brief Constants for the linear gain model - * - * \var CameraSensorHelper::AnalogueGainConstants::exp - * \brief Constants for the exponential gain model - */ - -/** * \var CameraSensorHelper::blackLevel_ * \brief The black level of the sensor * \sa CameraSensorHelper::blackLevel() */ /** - * \var CameraSensorHelper::gainType_ - * \brief The analogue gain model type - */ - -/** - * \var CameraSensorHelper::gainConstants_ + * \var CameraSensorHelper::gain_ * \brief The analogue gain parameters used for calculation * * The analogue gain is calculated through a formula, and its parameters are @@ -526,8 +480,7 @@ public: { /* From datasheet: 64 at 10bits. */ blackLevel_ = 4096; - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 100, 0, 0, 1024 }; + gain_ = AnalogueGainLinear{ 100, 0, 0, 1024 }; } }; REGISTER_CAMERA_SENSOR_HELPER("gc05a2", CameraSensorHelperGc05a2) @@ -539,8 +492,7 @@ public: { /* From datasheet: 64 at 10bits. */ blackLevel_ = 4096; - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 100, 0, 0, 1024 }; + gain_ = AnalogueGainLinear{ 100, 0, 0, 1024 }; } }; REGISTER_CAMERA_SENSOR_HELPER("gc08a3", CameraSensorHelperGc08a3) @@ -552,8 +504,7 @@ public: { /* From datasheet: 64 at 10bits. */ blackLevel_ = 4096; - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 0, 512, -1, 512 }; + gain_ = AnalogueGainLinear{ 0, 512, -1, 512 }; } }; REGISTER_CAMERA_SENSOR_HELPER("imx214", CameraSensorHelperImx214) @@ -565,8 +516,7 @@ public: { /* From datasheet: 64 at 10bits. */ blackLevel_ = 4096; - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 0, 256, -1, 256 }; + gain_ = AnalogueGainLinear{ 0, 256, -1, 256 }; } }; REGISTER_CAMERA_SENSOR_HELPER("imx219", CameraSensorHelperImx219) @@ -578,8 +528,7 @@ public: { /* From datasheet: 0x40 at 10bits. */ blackLevel_ = 4096; - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 0, 512, -1, 512 }; + gain_ = AnalogueGainLinear{ 0, 512, -1, 512 }; } }; REGISTER_CAMERA_SENSOR_HELPER("imx258", CameraSensorHelperImx258) @@ -591,8 +540,7 @@ public: { /* From datasheet: 0x32 at 10bits. */ blackLevel_ = 3200; - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 0, 2048, -1, 2048 }; + gain_ = AnalogueGainLinear{ 0, 2048, -1, 2048 }; } }; REGISTER_CAMERA_SENSOR_HELPER("imx283", CameraSensorHelperImx283) @@ -604,8 +552,7 @@ public: { /* From datasheet: 0xf0 at 12bits. */ blackLevel_ = 3840; - gainType_ = AnalogueGainExponential; - gainConstants_.exp = { 1.0, expGainDb(0.3) }; + gain_ = AnalogueGainExp{ 1.0, expGainDb(0.3) }; } }; REGISTER_CAMERA_SENSOR_HELPER("imx290", CameraSensorHelperImx290) @@ -615,8 +562,7 @@ class CameraSensorHelperImx296 : public CameraSensorHelper public: CameraSensorHelperImx296() { - gainType_ = AnalogueGainExponential; - gainConstants_.exp = { 1.0, expGainDb(0.1) }; + gain_ = AnalogueGainExp{ 1.0, expGainDb(0.1) }; } }; REGISTER_CAMERA_SENSOR_HELPER("imx296", CameraSensorHelperImx296) @@ -633,8 +579,7 @@ public: { /* From datasheet: 0x32 at 10bits. */ blackLevel_ = 3200; - gainType_ = AnalogueGainExponential; - gainConstants_.exp = { 1.0, expGainDb(0.3) }; + gain_ = AnalogueGainExp{ 1.0, expGainDb(0.3) }; } }; REGISTER_CAMERA_SENSOR_HELPER("imx335", CameraSensorHelperImx335) @@ -644,8 +589,7 @@ class CameraSensorHelperImx415 : public CameraSensorHelper public: CameraSensorHelperImx415() { - gainType_ = AnalogueGainExponential; - gainConstants_.exp = { 1.0, expGainDb(0.3) }; + gain_ = AnalogueGainExp{ 1.0, expGainDb(0.3) }; } }; REGISTER_CAMERA_SENSOR_HELPER("imx415", CameraSensorHelperImx415) @@ -660,8 +604,7 @@ class CameraSensorHelperImx477 : public CameraSensorHelper public: CameraSensorHelperImx477() { - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 0, 1024, -1, 1024 }; + gain_ = AnalogueGainLinear{ 0, 1024, -1, 1024 }; } }; REGISTER_CAMERA_SENSOR_HELPER("imx477", CameraSensorHelperImx477) @@ -675,8 +618,7 @@ public: * The Sensor Manual doesn't appear to document the gain model. * This has been validated with some empirical testing only. */ - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 128 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 128 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov2685", CameraSensorHelperOv2685) @@ -686,8 +628,7 @@ class CameraSensorHelperOv2740 : public CameraSensorHelper public: CameraSensorHelperOv2740() { - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 128 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 128 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov2740", CameraSensorHelperOv2740) @@ -699,8 +640,7 @@ public: { /* From datasheet: 0x40 at 12bits. */ blackLevel_ = 1024; - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 128 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 128 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov4689", CameraSensorHelperOv4689) @@ -712,8 +652,7 @@ public: { /* From datasheet: 0x10 at 10bits. */ blackLevel_ = 1024; - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 16 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 16 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov5640", CameraSensorHelperOv5640) @@ -723,8 +662,7 @@ class CameraSensorHelperOv5647 : public CameraSensorHelper public: CameraSensorHelperOv5647() { - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 16 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 16 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov5647", CameraSensorHelperOv5647) @@ -734,8 +672,7 @@ class CameraSensorHelperOv5670 : public CameraSensorHelper public: CameraSensorHelperOv5670() { - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 128 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 128 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov5670", CameraSensorHelperOv5670) @@ -747,8 +684,7 @@ public: { /* From Linux kernel driver: 0x40 at 10bits. */ blackLevel_ = 4096; - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 128 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 128 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov5675", CameraSensorHelperOv5675) @@ -758,8 +694,7 @@ class CameraSensorHelperOv5693 : public CameraSensorHelper public: CameraSensorHelperOv5693() { - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 16 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 16 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov5693", CameraSensorHelperOv5693) @@ -769,8 +704,7 @@ class CameraSensorHelperOv64a40 : public CameraSensorHelper public: CameraSensorHelperOv64a40() { - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 128 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 128 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov64a40", CameraSensorHelperOv64a40) @@ -780,15 +714,13 @@ class CameraSensorHelperOv8858 : public CameraSensorHelper public: CameraSensorHelperOv8858() { - gainType_ = AnalogueGainLinear; - /* * \todo Validate the selected 1/128 step value as it differs * from what the sensor manual describes. * * See: https://patchwork.linuxtv.org/project/linux-media/patch/20221106171129.166892-2-nicholas@rothemail.net/#142267 */ - gainConstants_.linear = { 1, 0, 0, 128 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 128 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov8858", CameraSensorHelperOv8858) @@ -798,8 +730,7 @@ class CameraSensorHelperOv8865 : public CameraSensorHelper public: CameraSensorHelperOv8865() { - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 128 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 128 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov8865", CameraSensorHelperOv8865) @@ -809,8 +740,7 @@ class CameraSensorHelperOv13858 : public CameraSensorHelper public: CameraSensorHelperOv13858() { - gainType_ = AnalogueGainLinear; - gainConstants_.linear = { 1, 0, 0, 128 }; + gain_ = AnalogueGainLinear{ 1, 0, 0, 128 }; } }; REGISTER_CAMERA_SENSOR_HELPER("ov13858", CameraSensorHelperOv13858) diff --git a/src/ipa/libipa/camera_sensor_helper.h b/src/ipa/libipa/camera_sensor_helper.h index 75868205..a9300a64 100644 --- a/src/ipa/libipa/camera_sensor_helper.h +++ b/src/ipa/libipa/camera_sensor_helper.h @@ -11,6 +11,7 @@ #include <optional> #include <stdint.h> #include <string> +#include <variant> #include <vector> #include <libcamera/base/class.h> @@ -30,31 +31,20 @@ public: virtual double gain(uint32_t gainCode) const; protected: - enum AnalogueGainType { - AnalogueGainLinear, - AnalogueGainExponential, - }; - - struct AnalogueGainLinearConstants { + struct AnalogueGainLinear { int16_t m0; int16_t c0; int16_t m1; int16_t c1; }; - struct AnalogueGainExpConstants { + struct AnalogueGainExp { double a; double m; }; - union AnalogueGainConstants { - AnalogueGainLinearConstants linear; - AnalogueGainExpConstants exp; - }; - std::optional<int16_t> blackLevel_; - AnalogueGainType gainType_; - AnalogueGainConstants gainConstants_; + std::variant<std::monostate, AnalogueGainLinear, AnalogueGainExp> gain_; private: LIBCAMERA_DISABLE_COPY_AND_MOVE(CameraSensorHelper) diff --git a/src/ipa/libipa/colours.h b/src/ipa/libipa/colours.h index fa6a8b57..d39b2ca8 100644 --- a/src/ipa/libipa/colours.h +++ b/src/ipa/libipa/colours.h @@ -9,7 +9,7 @@ #include <stdint.h> -#include "vector.h" +#include "libcamera/internal/vector.h" namespace libcamera { diff --git a/src/ipa/rkisp1/utils.cpp b/src/ipa/libipa/fixedpoint.cpp index 960ec64e..6b698fc5 100644 --- a/src/ipa/rkisp1/utils.cpp +++ b/src/ipa/libipa/fixedpoint.cpp @@ -2,18 +2,18 @@ /* * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com> * - * Miscellaneous utility functions specific to rkisp1 + * Fixed / floating point conversions */ -#include "utils.h" +#include "fixedpoint.h" /** - * \file utils.h + * \file fixedpoint.h */ namespace libcamera { -namespace ipa::rkisp1::utils { +namespace ipa { /** * \fn R floatingToFixedPoint(T number) @@ -37,6 +37,6 @@ namespace ipa::rkisp1::utils { * \return The converted value */ -} /* namespace ipa::rkisp1::utils */ +} /* namespace ipa */ } /* namespace libcamera */ diff --git a/src/ipa/rkisp1/utils.h b/src/ipa/libipa/fixedpoint.h index 5f38b50b..709cf50f 100644 --- a/src/ipa/rkisp1/utils.h +++ b/src/ipa/libipa/fixedpoint.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com> * - * Miscellaneous utility functions specific to rkisp1 + * Fixed / floating point conversions */ #pragma once @@ -12,7 +12,7 @@ namespace libcamera { -namespace ipa::rkisp1::utils { +namespace ipa { #ifndef __DOXYGEN__ template<unsigned int I, unsigned int F, typename R, typename T, @@ -60,6 +60,6 @@ constexpr R fixedToFloatingPoint(T number) return static_cast<R>(t) / static_cast<R>(1 << F); } -} /* namespace ipa::rkisp1::utils */ +} /* namespace ipa */ } /* namespace libcamera */ diff --git a/src/ipa/libipa/lux.cpp b/src/ipa/libipa/lux.cpp new file mode 100644 index 00000000..61f8fea8 --- /dev/null +++ b/src/ipa/libipa/lux.cpp @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi Ltd + * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com> + * + * Helper class that implements lux estimation + */ +#include "lux.h" + +#include <algorithm> +#include <chrono> + +#include <libcamera/base/log.h> + +#include "libcamera/internal/yaml_parser.h" + +#include "histogram.h" + +/** + * \file lux.h + * \brief Helper class that implements lux estimation + * + * Estimating the lux level of an image is a common operation that can for + * instance be used to adjust the target Y value in AGC or for Bayesian AWB + * estimation. + */ + +namespace libcamera { + +using namespace std::literals::chrono_literals; + +LOG_DEFINE_CATEGORY(Lux) + +namespace ipa { + +/** + * \class Lux + * \brief Class that implements lux estimation + * + * IPAs that wish to use lux estimation should create a Lux algorithm module + * that lightly wraps this module by providing the platform-specific luminance + * histogram. The Lux entry in the tuning file must then precede the algorithms + * that depend on the estimated lux value. + */ + +/** + * \var Lux::binSize_ + * \brief The maximum count of each bin + */ + +/** + * \var Lux::referenceExposureTime_ + * \brief The exposure time of the reference image, in microseconds + */ + +/** + * \var Lux::referenceAnalogueGain_ + * \brief The analogue gain of the reference image + */ + +/** + * \var Lux::referenceDigitalGain_ + * \brief The analogue gain of the reference image + */ + +/** + * \var Lux::referenceY_ + * \brief The measured luminance of the reference image, out of the bin size + * + * \sa binSize_ + */ + +/** + * \var Lux::referenceLux_ + * \brief The estimated lux level of the reference image + */ + +/** + * \brief Construct the Lux helper module + * \param[in] binSize The maximum count of each bin + */ +Lux::Lux(unsigned int binSize) + : binSize_(binSize) +{ +} + +/** + * \brief Parse tuning data + * \param[in] tuningData The YamlObject representing the tuning data + * + * This function parses yaml tuning data for the common Lux module. It requires + * reference exposure time, analogue gain, digital gain, and lux values. + * + * \code{.unparsed} + * algorithms: + * - Lux: + * referenceExposureTime: 10000 + * referenceAnalogueGain: 4.0 + * referenceDigitalGain: 1.0 + * referenceY: 12000 + * referenceLux: 1000 + * \endcode + * + * \return 0 on success or a negative error code + */ +int Lux::parseTuningData(const YamlObject &tuningData) +{ + auto value = tuningData["referenceExposureTime"].get<double>(); + if (!value) { + LOG(Lux, Error) << "Missing tuning parameter: " + << "'referenceExposureTime'"; + return -EINVAL; + } + referenceExposureTime_ = *value * 1.0us; + + value = tuningData["referenceAnalogueGain"].get<double>(); + if (!value) { + LOG(Lux, Error) << "Missing tuning parameter: " + << "'referenceAnalogueGain'"; + return -EINVAL; + } + referenceAnalogueGain_ = *value; + + value = tuningData["referenceDigitalGain"].get<double>(); + if (!value) { + LOG(Lux, Error) << "Missing tuning parameter: " + << "'referenceDigitalGain'"; + return -EINVAL; + } + referenceDigitalGain_ = *value; + + value = tuningData["referenceY"].get<double>(); + if (!value) { + LOG(Lux, Error) << "Missing tuning parameter: " + << "'referenceY'"; + return -EINVAL; + } + referenceY_ = *value; + + value = tuningData["referenceLux"].get<double>(); + if (!value) { + LOG(Lux, Error) << "Missing tuning parameter: " + << "'referenceLux'"; + return -EINVAL; + } + referenceLux_ = *value; + + return 0; +} + +/** + * \brief Estimate lux given runtime values + * \param[in] exposureTime Exposure time applied to the frame + * \param[in] aGain Analogue gain applied to the frame + * \param[in] dGain Digital gain applied to the frame + * \param[in] yHist Histogram from the ISP statistics + * + * Estimate the lux given the exposure time, gain, and histogram. + * + * \return Estimated lux value + */ +double Lux::estimateLux(utils::Duration exposureTime, + double aGain, double dGain, + const Histogram &yHist) const +{ + double currentY = yHist.interQuantileMean(0, 1); + double exposureTimeRatio = referenceExposureTime_ / exposureTime; + double aGainRatio = referenceAnalogueGain_ / aGain; + double dGainRatio = referenceDigitalGain_ / dGain; + double yRatio = currentY * (binSize_ / yHist.bins()) / referenceY_; + + double estimatedLux = exposureTimeRatio * aGainRatio * dGainRatio * + yRatio * referenceLux_; + + LOG(Lux, Debug) << "Estimated lux " << estimatedLux; + return estimatedLux; +} + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/lux.h b/src/ipa/libipa/lux.h new file mode 100644 index 00000000..93ca6479 --- /dev/null +++ b/src/ipa/libipa/lux.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi Ltd + * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com> + * + * Helper class that implements lux estimation + */ + +#pragma once + +#include <libcamera/base/utils.h> + +namespace libcamera { + +class YamlObject; + +namespace ipa { + +class Histogram; + +class Lux +{ +public: + Lux(unsigned int binSize); + + int parseTuningData(const YamlObject &tuningData); + double estimateLux(utils::Duration exposureTime, + double aGain, double dGain, + const Histogram &yHist) const; + +private: + unsigned int binSize_; + utils::Duration referenceExposureTime_; + double referenceAnalogueGain_; + double referenceDigitalGain_; + double referenceY_; + double referenceLux_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/meson.build b/src/ipa/libipa/meson.build index 93ae25da..12d8d15b 100644 --- a/src/ipa/libipa/meson.build +++ b/src/ipa/libipa/meson.build @@ -7,12 +7,13 @@ libipa_headers = files([ 'colours.h', 'exposure_mode_helper.h', 'fc_queue.h', + 'fixedpoint.h', 'histogram.h', 'interpolator.h', 'lsc_polynomial.h', + 'lux.h', 'module.h', 'pwl.h', - 'vector.h', ]) libipa_sources = files([ @@ -22,12 +23,13 @@ libipa_sources = files([ 'colours.cpp', 'exposure_mode_helper.cpp', 'fc_queue.cpp', + 'fixedpoint.cpp', 'histogram.cpp', 'interpolator.cpp', 'lsc_polynomial.cpp', + 'lux.cpp', 'module.cpp', 'pwl.cpp', - 'vector.cpp', ]) libipa_includes = include_directories('..') diff --git a/src/ipa/libipa/pwl.h b/src/ipa/libipa/pwl.h index d4ec9f4f..8fdc7053 100644 --- a/src/ipa/libipa/pwl.h +++ b/src/ipa/libipa/pwl.h @@ -12,7 +12,7 @@ #include <utility> #include <vector> -#include "vector.h" +#include "libcamera/internal/vector.h" namespace libcamera { diff --git a/src/ipa/libipa/vector.cpp b/src/ipa/libipa/vector.cpp deleted file mode 100644 index 8019f8cf..00000000 --- a/src/ipa/libipa/vector.cpp +++ /dev/null @@ -1,351 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1-or-later */ -/* - * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com> - * - * Vector and related operations - */ - -#include "vector.h" - -#include <libcamera/base/log.h> - -/** - * \file vector.h - * \brief Vector class - */ - -namespace libcamera { - -LOG_DEFINE_CATEGORY(Vector) - -namespace ipa { - -/** - * \class Vector - * \brief Vector class - * \tparam T Type of numerical values to be stored in the vector - * \tparam Rows Number of dimension of the vector (= number of elements) - */ - -/** - * \fn Vector::Vector() - * \brief Construct an uninitialized vector - */ - -/** - * \fn Vector::Vector(T scalar) - * \brief Construct a vector filled with a \a scalar value - * \param[in] scalar The scalar value - */ - -/** - * \fn Vector::Vector(const std::array<T, Rows> &data) - * \brief Construct vector from supplied data - * \param data Data from which to construct a vector - * - * The size of \a data must be equal to the dimension size Rows of the vector. - */ - -/** - * \fn T Vector::operator[](size_t i) const - * \brief Index to an element in the vector - * \param i Index of element to retrieve - * \return Element at index \a i from the vector - */ - -/** - * \fn T &Vector::operator[](size_t i) - * \copydoc Vector::operator[](size_t i) const - */ - -/** - * \fn Vector::operator-() const - * \brief Negate a Vector by negating both all of its coordinates - * \return The negated vector - */ - -/** - * \fn Vector::operator+(Vector const &other) const - * \brief Calculate the sum of this vector and \a other element-wise - * \param[in] other The other vector - * \return The element-wise sum of this vector and \a other - */ - -/** - * \fn Vector::operator+(T scalar) const - * \brief Calculate the sum of this vector and \a scalar element-wise - * \param[in] scalar The scalar - * \return The element-wise sum of this vector and \a other - */ - -/** - * \fn Vector::operator-(Vector const &other) const - * \brief Calculate the difference of this vector and \a other element-wise - * \param[in] other The other vector - * \return The element-wise subtraction of \a other from this vector - */ - -/** - * \fn Vector::operator-(T scalar) const - * \brief Calculate the difference of this vector and \a scalar element-wise - * \param[in] scalar The scalar - * \return The element-wise subtraction of \a scalar from this vector - */ - -/** - * \fn Vector::operator*(const Vector &other) const - * \brief Calculate the product of this vector and \a other element-wise - * \param[in] other The other vector - * \return The element-wise product of this vector and \a other - */ - -/** - * \fn Vector::operator*(T scalar) const - * \brief Calculate the product of this vector and \a scalar element-wise - * \param[in] scalar The scalar - * \return The element-wise product of this vector and \a scalar - */ - -/** - * \fn Vector::operator/(const Vector &other) const - * \brief Calculate the quotient of this vector and \a other element-wise - * \param[in] other The other vector - * \return The element-wise division of this vector by \a other - */ - -/** - * \fn Vector::operator/(T scalar) const - * \brief Calculate the quotient of this vector and \a scalar element-wise - * \param[in] scalar The scalar - * \return The element-wise division of this vector by \a scalar - */ - -/** - * \fn Vector::operator+=(Vector const &other) - * \brief Add \a other element-wise to this vector - * \param[in] other The other vector - * \return This vector - */ - -/** - * \fn Vector::operator+=(T scalar) - * \brief Add \a scalar element-wise to this vector - * \param[in] scalar The scalar - * \return This vector - */ - -/** - * \fn Vector::operator-=(Vector const &other) - * \brief Subtract \a other element-wise from this vector - * \param[in] other The other vector - * \return This vector - */ - -/** - * \fn Vector::operator-=(T scalar) - * \brief Subtract \a scalar element-wise from this vector - * \param[in] scalar The scalar - * \return This vector - */ - -/** - * \fn Vector::operator*=(const Vector &other) - * \brief Multiply this vector by \a other element-wise - * \param[in] other The other vector - * \return This vector - */ - -/** - * \fn Vector::operator*=(T scalar) - * \brief Multiply this vector by \a scalar element-wise - * \param[in] scalar The scalar - * \return This vector - */ - -/** - * \fn Vector::operator/=(const Vector &other) - * \brief Divide this vector by \a other element-wise - * \param[in] other The other vector - * \return This vector - */ - -/** - * \fn Vector::operator/=(T scalar) - * \brief Divide this vector by \a scalar element-wise - * \param[in] scalar The scalar - * \return This vector - */ - -/** - * \fn Vector::min(const Vector &other) const - * \brief Calculate the minimum of this vector and \a other element-wise - * \param[in] other The other vector - * \return The element-wise minimum of this vector and \a other - */ - -/** - * \fn Vector::min(T scalar) const - * \brief Calculate the minimum of this vector and \a scalar element-wise - * \param[in] scalar The scalar - * \return The element-wise minimum of this vector and \a scalar - */ - -/** - * \fn Vector::max(const Vector &other) const - * \brief Calculate the maximum of this vector and \a other element-wise - * \param[in] other The other vector - * \return The element-wise maximum of this vector and \a other - */ - -/** - * \fn Vector::max(T scalar) const - * \brief Calculate the maximum of this vector and \a scalar element-wise - * \param[in] scalar The scalar - * \return The element-wise maximum of this vector and \a scalar - */ - -/** - * \fn Vector::dot(const Vector<T, Rows> &other) const - * \brief Compute the dot product - * \param[in] other The other vector - * \return The dot product of the two vectors - */ - -/** - * \fn constexpr T &Vector::x() - * \brief Convenience function to access the first element of the vector - * \return The first element of the vector - */ - -/** - * \fn constexpr T &Vector::y() - * \brief Convenience function to access the second element of the vector - * \return The second element of the vector - */ - -/** - * \fn constexpr T &Vector::z() - * \brief Convenience function to access the third element of the vector - * \return The third element of the vector - */ - -/** - * \fn constexpr const T &Vector::x() const - * \copydoc Vector::x() - */ - -/** - * \fn constexpr const T &Vector::y() const - * \copydoc Vector::y() - */ - -/** - * \fn constexpr const T &Vector::z() const - * \copydoc Vector::z() - */ - -/** - * \fn constexpr T &Vector::r() - * \brief Convenience function to access the first element of the vector - * \return The first element of the vector - */ - -/** - * \fn constexpr T &Vector::g() - * \brief Convenience function to access the second element of the vector - * \return The second element of the vector - */ - -/** - * \fn constexpr T &Vector::b() - * \brief Convenience function to access the third element of the vector - * \return The third element of the vector - */ - -/** - * \fn constexpr const T &Vector::r() const - * \copydoc Vector::r() - */ - -/** - * \fn constexpr const T &Vector::g() const - * \copydoc Vector::g() - */ - -/** - * \fn constexpr const T &Vector::b() const - * \copydoc Vector::b() - */ - -/** - * \fn Vector::length2() - * \brief Get the squared length of the vector - * \return The squared length of the vector - */ - -/** - * \fn Vector::length() - * \brief Get the length of the vector - * \return The length of the vector - */ - -/** - * \fn Vector::sum() const - * \brief Calculate the sum of all the vector elements - * \tparam R The type of the sum - * - * The type R of the sum defaults to the type T of the elements, but can be set - * explicitly to use a different type in case the type T would risk - * overflowing. - * - * \return The sum of all the vector elements - */ - -/** - * \fn Vector<T, Rows> operator*(const Matrix<T, Rows, Cols> &m, const Vector<T, Cols> &v) - * \brief Multiply a matrix by a vector - * \tparam T Numerical type of the contents of the matrix and vector - * \tparam Rows The number of rows in the matrix - * \tparam Cols The number of columns in the matrix (= rows in the vector) - * \param m The matrix - * \param v The vector - * \return Product of matrix \a m and vector \a v - */ - -/** - * \typedef RGB - * \brief A Vector of 3 elements representing an RGB pixel value - */ - -/** - * \fn bool operator==(const Vector<T, Rows> &lhs, const Vector<T, Rows> &rhs) - * \brief Compare vectors for equality - * \return True if the two vectors are equal, false otherwise - */ - -/** - * \fn bool operator!=(const Vector<T, Rows> &lhs, const Vector<T, Rows> &rhs) - * \brief Compare vectors for inequality - * \return True if the two vectors are not equal, false otherwise - */ - -#ifndef __DOXYGEN__ -bool vectorValidateYaml(const YamlObject &obj, unsigned int size) -{ - if (!obj.isList()) - return false; - - if (obj.size() != size) { - LOG(Vector, Error) - << "Wrong number of values in YAML vector: expected " - << size << ", got " << obj.size(); - return false; - } - - return true; -} -#endif /* __DOXYGEN__ */ - -} /* namespace ipa */ - -} /* namespace libcamera */ diff --git a/src/ipa/libipa/vector.h b/src/ipa/libipa/vector.h deleted file mode 100644 index fe33c9d6..00000000 --- a/src/ipa/libipa/vector.h +++ /dev/null @@ -1,370 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1-or-later */ -/* - * Copyright (C) 2024, Paul Elder <paul.elder@ideasonboard.com> - * - * Vector and related operations - */ -#pragma once - -#include <algorithm> -#include <array> -#include <cmath> -#include <functional> -#include <numeric> -#include <optional> -#include <ostream> - -#include <libcamera/base/log.h> -#include <libcamera/base/span.h> - -#include "libcamera/internal/matrix.h" -#include "libcamera/internal/yaml_parser.h" - -namespace libcamera { - -LOG_DECLARE_CATEGORY(Vector) - -namespace ipa { - -#ifndef __DOXYGEN__ -template<typename T, unsigned int Rows, - std::enable_if_t<std::is_arithmetic_v<T>> * = nullptr> -#else -template<typename T, unsigned int Rows> -#endif /* __DOXYGEN__ */ -class Vector -{ -public: - constexpr Vector() = default; - - constexpr explicit Vector(T scalar) - { - data_.fill(scalar); - } - - constexpr Vector(const std::array<T, Rows> &data) - { - for (unsigned int i = 0; i < Rows; i++) - data_[i] = data[i]; - } - - const T &operator[](size_t i) const - { - ASSERT(i < data_.size()); - return data_[i]; - } - - T &operator[](size_t i) - { - ASSERT(i < data_.size()); - return data_[i]; - } - - constexpr Vector<T, Rows> operator-() const - { - Vector<T, Rows> ret; - for (unsigned int i = 0; i < Rows; i++) - ret[i] = -data_[i]; - return ret; - } - - constexpr Vector operator+(const Vector &other) const - { - return apply(*this, other, std::plus<>{}); - } - - constexpr Vector operator+(T scalar) const - { - return apply(*this, scalar, std::plus<>{}); - } - - constexpr Vector operator-(const Vector &other) const - { - return apply(*this, other, std::minus<>{}); - } - - constexpr Vector operator-(T scalar) const - { - return apply(*this, scalar, std::minus<>{}); - } - - constexpr Vector operator*(const Vector &other) const - { - return apply(*this, other, std::multiplies<>{}); - } - - constexpr Vector operator*(T scalar) const - { - return apply(*this, scalar, std::multiplies<>{}); - } - - constexpr Vector operator/(const Vector &other) const - { - return apply(*this, other, std::divides<>{}); - } - - constexpr Vector operator/(T scalar) const - { - return apply(*this, scalar, std::divides<>{}); - } - - Vector &operator+=(const Vector &other) - { - return apply(other, [](T a, T b) { return a + b; }); - } - - Vector &operator+=(T scalar) - { - return apply(scalar, [](T a, T b) { return a + b; }); - } - - Vector &operator-=(const Vector &other) - { - return apply(other, [](T a, T b) { return a - b; }); - } - - Vector &operator-=(T scalar) - { - return apply(scalar, [](T a, T b) { return a - b; }); - } - - Vector &operator*=(const Vector &other) - { - return apply(other, [](T a, T b) { return a * b; }); - } - - Vector &operator*=(T scalar) - { - return apply(scalar, [](T a, T b) { return a * b; }); - } - - Vector &operator/=(const Vector &other) - { - return apply(other, [](T a, T b) { return a / b; }); - } - - Vector &operator/=(T scalar) - { - return apply(scalar, [](T a, T b) { return a / b; }); - } - - constexpr Vector min(const Vector &other) const - { - return apply(*this, other, [](T a, T b) { return std::min(a, b); }); - } - - constexpr Vector min(T scalar) const - { - return apply(*this, scalar, [](T a, T b) { return std::min(a, b); }); - } - - constexpr Vector max(const Vector &other) const - { - return apply(*this, other, [](T a, T b) { return std::max(a, b); }); - } - - constexpr Vector max(T scalar) const - { - return apply(*this, scalar, [](T a, T b) -> T { return std::max(a, b); }); - } - - constexpr T dot(const Vector<T, Rows> &other) const - { - T ret = 0; - for (unsigned int i = 0; i < Rows; i++) - ret += data_[i] * other[i]; - return ret; - } - -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 1>> -#endif /* __DOXYGEN__ */ - constexpr const T &x() const { return data_[0]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 2>> -#endif /* __DOXYGEN__ */ - constexpr const T &y() const { return data_[1]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 3>> -#endif /* __DOXYGEN__ */ - constexpr const T &z() const { return data_[2]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 1>> -#endif /* __DOXYGEN__ */ - constexpr T &x() { return data_[0]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 2>> -#endif /* __DOXYGEN__ */ - constexpr T &y() { return data_[1]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 3>> -#endif /* __DOXYGEN__ */ - constexpr T &z() { return data_[2]; } - -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 1>> -#endif /* __DOXYGEN__ */ - constexpr const T &r() const { return data_[0]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 2>> -#endif /* __DOXYGEN__ */ - constexpr const T &g() const { return data_[1]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 3>> -#endif /* __DOXYGEN__ */ - constexpr const T &b() const { return data_[2]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 1>> -#endif /* __DOXYGEN__ */ - constexpr T &r() { return data_[0]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 2>> -#endif /* __DOXYGEN__ */ - constexpr T &g() { return data_[1]; } -#ifndef __DOXYGEN__ - template<bool Dependent = false, typename = std::enable_if_t<Dependent || Rows >= 3>> -#endif /* __DOXYGEN__ */ - constexpr T &b() { return data_[2]; } - - constexpr double length2() const - { - double ret = 0; - for (unsigned int i = 0; i < Rows; i++) - ret += data_[i] * data_[i]; - return ret; - } - - constexpr double length() const - { - return std::sqrt(length2()); - } - - template<typename R = T> - constexpr R sum() const - { - return std::accumulate(data_.begin(), data_.end(), R{}); - } - -private: - template<class BinaryOp> - static constexpr Vector apply(const Vector &lhs, const Vector &rhs, BinaryOp op) - { - Vector result; - std::transform(lhs.data_.begin(), lhs.data_.end(), - rhs.data_.begin(), result.data_.begin(), - op); - - return result; - } - - template<class BinaryOp> - static constexpr Vector apply(const Vector &lhs, T rhs, BinaryOp op) - { - Vector result; - std::transform(lhs.data_.begin(), lhs.data_.end(), - result.data_.begin(), - [&op, rhs](T v) { return op(v, rhs); }); - - return result; - } - - template<class BinaryOp> - Vector &apply(const Vector &other, BinaryOp op) - { - auto itOther = other.data_.begin(); - std::for_each(data_.begin(), data_.end(), - [&op, &itOther](T &v) { v = op(v, *itOther++); }); - - return *this; - } - - template<class BinaryOp> - Vector &apply(T scalar, BinaryOp op) - { - std::for_each(data_.begin(), data_.end(), - [&op, scalar](T &v) { v = op(v, scalar); }); - - return *this; - } - - std::array<T, Rows> data_; -}; - -template<typename T> -using RGB = Vector<T, 3>; - -template<typename T, unsigned int Rows, unsigned int Cols> -Vector<T, Rows> operator*(const Matrix<T, Rows, Cols> &m, const Vector<T, Cols> &v) -{ - Vector<T, Rows> result; - - for (unsigned int i = 0; i < Rows; i++) { - T sum = 0; - for (unsigned int j = 0; j < Cols; j++) - sum += m[i][j] * v[j]; - result[i] = sum; - } - - return result; -} - -template<typename T, unsigned int Rows> -bool operator==(const Vector<T, Rows> &lhs, const Vector<T, Rows> &rhs) -{ - for (unsigned int i = 0; i < Rows; i++) { - if (lhs[i] != rhs[i]) - return false; - } - - return true; -} - -template<typename T, unsigned int Rows> -bool operator!=(const Vector<T, Rows> &lhs, const Vector<T, Rows> &rhs) -{ - return !(lhs == rhs); -} - -#ifndef __DOXYGEN__ -bool vectorValidateYaml(const YamlObject &obj, unsigned int size); -#endif /* __DOXYGEN__ */ - -} /* namespace ipa */ - -#ifndef __DOXYGEN__ -template<typename T, unsigned int Rows> -std::ostream &operator<<(std::ostream &out, const ipa::Vector<T, Rows> &v) -{ - out << "Vector { "; - for (unsigned int i = 0; i < Rows; i++) { - out << v[i]; - out << ((i + 1 < Rows) ? ", " : " "); - } - out << " }"; - - return out; -} - -template<typename T, unsigned int Rows> -struct YamlObject::Getter<ipa::Vector<T, Rows>> { - std::optional<ipa::Vector<T, Rows>> get(const YamlObject &obj) const - { - if (!ipa::vectorValidateYaml(obj, Rows)) - return std::nullopt; - - ipa::Vector<T, Rows> vector; - - unsigned int i = 0; - for (const YamlObject &entry : obj.asList()) { - const auto value = entry.get<T>(); - if (!value) - return std::nullopt; - vector[i++] = *value; - } - - return vector; - } -}; -#endif /* __DOXYGEN__ */ - -} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/agc.cpp b/src/ipa/mali-c55/algorithms/agc.cpp new file mode 100644 index 00000000..70667db3 --- /dev/null +++ b/src/ipa/mali-c55/algorithms/agc.cpp @@ -0,0 +1,410 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board Oy + * + * agc.cpp - AGC/AEC mean-based control algorithm + */ + +#include "agc.h" + +#include <cmath> + +#include <libcamera/base/log.h> +#include <libcamera/base/utils.h> + +#include <libcamera/control_ids.h> +#include <libcamera/property_ids.h> + +#include "libipa/colours.h" +#include "libipa/fixedpoint.h" + +namespace libcamera { + +using namespace std::literals::chrono_literals; + +namespace ipa::mali_c55::algorithms { + +LOG_DEFINE_CATEGORY(MaliC55Agc) + +/* + * Number of histogram bins. This is only true for the specific configuration we + * set to the ISP; 4 separate histograms of 256 bins each. If that configuration + * ever changes then this constant will need updating. + */ +static constexpr unsigned int kNumHistogramBins = 256; + +/* + * The Mali-C55 ISP has a digital gain block which allows setting gain in Q5.8 + * format, a range of 0.0 to (very nearly) 32.0. We clamp from 1.0 to the actual + * max value which is 8191 * 2^-8. + */ +static constexpr double kMinDigitalGain = 1.0; +static constexpr double kMaxDigitalGain = 31.99609375; + +uint32_t AgcStatistics::decodeBinValue(uint16_t binVal) +{ + int exponent = (binVal & 0xf000) >> 12; + int mantissa = binVal & 0xfff; + + if (!exponent) + return mantissa * 2; + else + return (mantissa + 4096) * std::pow(2, exponent); +} + +/* + * We configure the ISP to give us 4 histograms of 256 bins each, with + * a single histogram per colour channel (R/Gr/Gb/B). The memory space + * containing the data is a single block containing all 4 histograms + * with the position of each colour's histogram within it dependent on + * the bayer pattern of the data input to the ISP. + * + * NOTE: The validity of this function depends on the parameters we have + * configured. With different skip/offset x, y values not all of the + * colour channels would be populated, and they may not be in the same + * planes as calculated here. + */ +int AgcStatistics::setBayerOrderIndices(BayerFormat::Order bayerOrder) +{ + switch (bayerOrder) { + case BayerFormat::Order::RGGB: + rIndex_ = 0; + grIndex_ = 1; + gbIndex_ = 2; + bIndex_ = 3; + break; + case BayerFormat::Order::GRBG: + grIndex_ = 0; + rIndex_ = 1; + bIndex_ = 2; + gbIndex_ = 3; + break; + case BayerFormat::Order::GBRG: + gbIndex_ = 0; + bIndex_ = 1; + rIndex_ = 2; + grIndex_ = 3; + break; + case BayerFormat::Order::BGGR: + bIndex_ = 0; + gbIndex_ = 1; + grIndex_ = 2; + rIndex_ = 3; + break; + default: + LOG(MaliC55Agc, Error) + << "Invalid bayer format " << bayerOrder; + return -EINVAL; + } + + return 0; +} + +void AgcStatistics::parseStatistics(const mali_c55_stats_buffer *stats) +{ + uint32_t r[256], g[256], b[256], y[256]; + + /* + * We need to decode the bin values for each histogram from their 16-bit + * compressed values to a 32-bit value. We also take the average of the + * Gr/Gb values into a single green histogram. + */ + for (unsigned int i = 0; i < 256; i++) { + r[i] = decodeBinValue(stats->ae_1024bin_hist.bins[i + (256 * rIndex_)]); + g[i] = (decodeBinValue(stats->ae_1024bin_hist.bins[i + (256 * grIndex_)]) + + decodeBinValue(stats->ae_1024bin_hist.bins[i + (256 * gbIndex_)])) / 2; + b[i] = decodeBinValue(stats->ae_1024bin_hist.bins[i + (256 * bIndex_)]); + + y[i] = rec601LuminanceFromRGB({ { static_cast<double>(r[i]), + static_cast<double>(g[i]), + static_cast<double>(b[i]) } }); + } + + rHist = Histogram(Span<uint32_t>(r, kNumHistogramBins)); + gHist = Histogram(Span<uint32_t>(g, kNumHistogramBins)); + bHist = Histogram(Span<uint32_t>(b, kNumHistogramBins)); + yHist = Histogram(Span<uint32_t>(y, kNumHistogramBins)); +} + +Agc::Agc() + : AgcMeanLuminance() +{ +} + +int Agc::init(IPAContext &context, const YamlObject &tuningData) +{ + int ret = parseTuningData(tuningData); + if (ret) + return ret; + + context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true); + context.ctrlMap[&controls::DigitalGain] = ControlInfo( + static_cast<float>(kMinDigitalGain), + static_cast<float>(kMaxDigitalGain), + static_cast<float>(kMinDigitalGain) + ); + context.ctrlMap.merge(controls()); + + return 0; +} + +int Agc::configure(IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + int ret = statistics_.setBayerOrderIndices(context.configuration.sensor.bayerOrder); + if (ret) + return ret; + + /* + * Defaults; we use whatever the sensor's default exposure is and the + * minimum analogue gain. AEGC is _active_ by default. + */ + context.activeState.agc.autoEnabled = true; + context.activeState.agc.automatic.sensorGain = context.configuration.agc.minAnalogueGain; + context.activeState.agc.automatic.exposure = context.configuration.agc.defaultExposure; + context.activeState.agc.automatic.ispGain = kMinDigitalGain; + context.activeState.agc.manual.sensorGain = context.configuration.agc.minAnalogueGain; + context.activeState.agc.manual.exposure = context.configuration.agc.defaultExposure; + context.activeState.agc.manual.ispGain = kMinDigitalGain; + context.activeState.agc.constraintMode = constraintModes().begin()->first; + context.activeState.agc.exposureMode = exposureModeHelpers().begin()->first; + + /* \todo Run this again when FrameDurationLimits is passed in */ + setLimits(context.configuration.agc.minShutterSpeed, + context.configuration.agc.maxShutterSpeed, + context.configuration.agc.minAnalogueGain, + context.configuration.agc.maxAnalogueGain); + + resetFrameCount(); + + return 0; +} + +void Agc::queueRequest(IPAContext &context, const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + const ControlList &controls) +{ + auto &agc = context.activeState.agc; + + const auto &constraintMode = controls.get(controls::AeConstraintMode); + agc.constraintMode = constraintMode.value_or(agc.constraintMode); + + const auto &exposureMode = controls.get(controls::AeExposureMode); + agc.exposureMode = exposureMode.value_or(agc.exposureMode); + + const auto &agcEnable = controls.get(controls::AeEnable); + if (agcEnable && *agcEnable != agc.autoEnabled) { + agc.autoEnabled = *agcEnable; + + LOG(MaliC55Agc, Info) + << (agc.autoEnabled ? "Enabling" : "Disabling") + << " AGC"; + } + + /* + * If the automatic exposure and gain is enabled we have no further work + * to do here... + */ + if (agc.autoEnabled) + return; + + /* + * ...otherwise we need to look for exposure and gain controls and use + * those to set the activeState. + */ + const auto &exposure = controls.get(controls::ExposureTime); + if (exposure) { + agc.manual.exposure = *exposure * 1.0us / context.configuration.sensor.lineDuration; + + LOG(MaliC55Agc, Debug) + << "Exposure set to " << agc.manual.exposure + << " on request sequence " << frame; + } + + const auto &analogueGain = controls.get(controls::AnalogueGain); + if (analogueGain) { + agc.manual.sensorGain = *analogueGain; + + LOG(MaliC55Agc, Debug) + << "Analogue gain set to " << agc.manual.sensorGain + << " on request sequence " << frame; + } + + const auto &digitalGain = controls.get(controls::DigitalGain); + if (digitalGain) { + agc.manual.ispGain = *digitalGain; + + LOG(MaliC55Agc, Debug) + << "Digital gain set to " << agc.manual.ispGain + << " on request sequence " << frame; + } +} + +size_t Agc::fillGainParamBlock(IPAContext &context, IPAFrameContext &frameContext, + mali_c55_params_block block) +{ + IPAActiveState &activeState = context.activeState; + double gain; + + if (activeState.agc.autoEnabled) + gain = activeState.agc.automatic.ispGain; + else + gain = activeState.agc.manual.ispGain; + + block.header->type = MALI_C55_PARAM_BLOCK_DIGITAL_GAIN; + block.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE; + block.header->size = sizeof(struct mali_c55_params_digital_gain); + + block.digital_gain->gain = floatingToFixedPoint<5, 8, uint16_t, double>(gain); + frameContext.agc.ispGain = gain; + + return block.header->size; +} + +size_t Agc::fillParamsBuffer(mali_c55_params_block block, + enum mali_c55_param_block_type type) +{ + block.header->type = type; + block.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE; + block.header->size = sizeof(struct mali_c55_params_aexp_hist); + + /* Collect every 3rd pixel horizontally */ + block.aexp_hist->skip_x = 1; + /* Start from first column */ + block.aexp_hist->offset_x = 0; + /* Collect every pixel vertically */ + block.aexp_hist->skip_y = 0; + /* Start from the first row */ + block.aexp_hist->offset_y = 0; + /* 1x scaling (i.e. none) */ + block.aexp_hist->scale_bottom = 0; + block.aexp_hist->scale_top = 0; + /* Collect all Bayer planes into 4 separate histograms */ + block.aexp_hist->plane_mode = 1; + /* Tap the data immediately after the digital gain block */ + block.aexp_hist->tap_point = MALI_C55_AEXP_HIST_TAP_FS; + + return block.header->size; +} + +size_t Agc::fillWeightsArrayBuffer(mali_c55_params_block block, + enum mali_c55_param_block_type type) +{ + block.header->type = type; + block.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE; + block.header->size = sizeof(struct mali_c55_params_aexp_weights); + + /* We use every zone - a 15x15 grid */ + block.aexp_weights->nodes_used_horiz = 15; + block.aexp_weights->nodes_used_vert = 15; + + /* + * We uniformly weight the zones to 1 - this results in the collected + * histograms containing a true pixel count, which we can then use to + * approximate colour channel averages for the image. + */ + Span<uint8_t> weights{ + block.aexp_weights->zone_weights, + MALI_C55_MAX_ZONES + }; + std::fill(weights.begin(), weights.end(), 1); + + return block.header->size; +} + +void Agc::prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, mali_c55_params_buffer *params) +{ + mali_c55_params_block block; + + block.data = ¶ms->data[params->total_size]; + params->total_size += fillGainParamBlock(context, frameContext, block); + + if (frame > 0) + return; + + block.data = ¶ms->data[params->total_size]; + params->total_size += fillParamsBuffer(block, + MALI_C55_PARAM_BLOCK_AEXP_HIST); + + block.data = ¶ms->data[params->total_size]; + params->total_size += fillWeightsArrayBuffer(block, + MALI_C55_PARAM_BLOCK_AEXP_HIST_WEIGHTS); + + block.data = ¶ms->data[params->total_size]; + params->total_size += fillParamsBuffer(block, + MALI_C55_PARAM_BLOCK_AEXP_IHIST); + + block.data = ¶ms->data[params->total_size]; + params->total_size += fillWeightsArrayBuffer(block, + MALI_C55_PARAM_BLOCK_AEXP_IHIST_WEIGHTS); +} + +double Agc::estimateLuminance(const double gain) const +{ + double rAvg = statistics_.rHist.interQuantileMean(0, 1) * gain; + double gAvg = statistics_.gHist.interQuantileMean(0, 1) * gain; + double bAvg = statistics_.bHist.interQuantileMean(0, 1) * gain; + double yAvg = rec601LuminanceFromRGB({ { rAvg, gAvg, bAvg } }); + + return yAvg / kNumHistogramBins; +} + +void Agc::process(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const mali_c55_stats_buffer *stats, + [[maybe_unused]] ControlList &metadata) +{ + IPASessionConfiguration &configuration = context.configuration; + IPAActiveState &activeState = context.activeState; + + if (!stats) { + LOG(MaliC55Agc, Error) << "No statistics buffer passed to Agc"; + return; + } + + statistics_.parseStatistics(stats); + context.activeState.agc.temperatureK = estimateCCT({ { statistics_.rHist.interQuantileMean(0, 1), + statistics_.gHist.interQuantileMean(0, 1), + statistics_.bHist.interQuantileMean(0, 1) } }); + + /* + * The Agc algorithm needs to know the effective exposure value that was + * applied to the sensor when the statistics were collected. + */ + uint32_t exposure = frameContext.agc.exposure; + double analogueGain = frameContext.agc.sensorGain; + double digitalGain = frameContext.agc.ispGain; + double totalGain = analogueGain * digitalGain; + utils::Duration currentShutter = exposure * configuration.sensor.lineDuration; + utils::Duration effectiveExposureValue = currentShutter * totalGain; + + utils::Duration shutterTime; + double aGain, dGain; + std::tie(shutterTime, aGain, dGain) = + calculateNewEv(activeState.agc.constraintMode, + activeState.agc.exposureMode, statistics_.yHist, + effectiveExposureValue); + + dGain = std::clamp(dGain, kMinDigitalGain, kMaxDigitalGain); + + LOG(MaliC55Agc, Debug) + << "Divided up shutter, analogue gain and digital gain are " + << shutterTime << ", " << aGain << " and " << dGain; + + activeState.agc.automatic.exposure = shutterTime / configuration.sensor.lineDuration; + activeState.agc.automatic.sensorGain = aGain; + activeState.agc.automatic.ispGain = dGain; + + metadata.set(controls::ExposureTime, currentShutter.get<std::micro>()); + metadata.set(controls::AnalogueGain, frameContext.agc.sensorGain); + metadata.set(controls::DigitalGain, frameContext.agc.ispGain); + metadata.set(controls::ColourTemperature, context.activeState.agc.temperatureK); +} + +REGISTER_IPA_ALGORITHM(Agc, "Agc") + +} /* namespace ipa::mali_c55::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/agc.h b/src/ipa/mali-c55/algorithms/agc.h new file mode 100644 index 00000000..c5c574e5 --- /dev/null +++ b/src/ipa/mali-c55/algorithms/agc.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2023, Ideas on Board Oy + * + * agc.h - Mali C55 AGC/AEC mean-based control algorithm + */ + +#pragma once + +#include <libcamera/base/utils.h> + +#include "libcamera/internal/bayer_format.h" + +#include "libipa/agc_mean_luminance.h" +#include "libipa/histogram.h" + +#include "algorithm.h" +#include "ipa_context.h" + +namespace libcamera { + +namespace ipa::mali_c55::algorithms { + +class AgcStatistics +{ +public: + AgcStatistics() + { + } + + int setBayerOrderIndices(BayerFormat::Order bayerOrder); + uint32_t decodeBinValue(uint16_t binVal); + void parseStatistics(const mali_c55_stats_buffer *stats); + + Histogram rHist; + Histogram gHist; + Histogram bHist; + Histogram yHist; +private: + unsigned int rIndex_; + unsigned int grIndex_; + unsigned int gbIndex_; + unsigned int bIndex_; +}; + +class Agc : public Algorithm, public AgcMeanLuminance +{ +public: + Agc(); + ~Agc() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) override; + void queueRequest(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const ControlList &controls) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + mali_c55_params_buffer *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const mali_c55_stats_buffer *stats, + ControlList &metadata) override; + +private: + double estimateLuminance(const double gain) const override; + size_t fillGainParamBlock(IPAContext &context, + IPAFrameContext &frameContext, + mali_c55_params_block block); + size_t fillParamsBuffer(mali_c55_params_block block, + enum mali_c55_param_block_type type); + size_t fillWeightsArrayBuffer(mali_c55_params_block block, + enum mali_c55_param_block_type type); + + AgcStatistics statistics_; +}; + +} /* namespace ipa::mali_c55::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/algorithm.h b/src/ipa/mali-c55/algorithms/algorithm.h new file mode 100644 index 00000000..36a3bff0 --- /dev/null +++ b/src/ipa/mali-c55/algorithms/algorithm.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * algorithm.h - Mali-C55 control algorithm interface + */ + +#pragma once + +#include <linux/mali-c55-config.h> + +#include <libipa/algorithm.h> + +#include "module.h" + +namespace libcamera { + +namespace ipa::mali_c55 { + +class Algorithm : public libcamera::ipa::Algorithm<Module> +{ +}; + +union mali_c55_params_block { + struct mali_c55_params_block_header *header; + struct mali_c55_params_sensor_off_preshading *sensor_offs; + struct mali_c55_params_aexp_hist *aexp_hist; + struct mali_c55_params_aexp_weights *aexp_weights; + struct mali_c55_params_digital_gain *digital_gain; + struct mali_c55_params_awb_gains *awb_gains; + struct mali_c55_params_awb_config *awb_config; + struct mali_c55_params_mesh_shading_config *shading_config; + struct mali_c55_params_mesh_shading_selection *shading_selection; + __u8 *data; +}; + +} /* namespace ipa::mali_c55 */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/awb.cpp b/src/ipa/mali-c55/algorithms/awb.cpp new file mode 100644 index 00000000..050b191b --- /dev/null +++ b/src/ipa/mali-c55/algorithms/awb.cpp @@ -0,0 +1,230 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board Oy + * + * awb.cpp - Mali C55 grey world auto white balance algorithm + */ + +#include "awb.h" + +#include <cmath> + +#include <libcamera/base/log.h> +#include <libcamera/base/utils.h> + +#include <libcamera/control_ids.h> + +#include "libipa/fixedpoint.h" + +namespace libcamera { + +namespace ipa::mali_c55::algorithms { + +LOG_DEFINE_CATEGORY(MaliC55Awb) + +/* Number of frames at which we should run AWB at full speed */ +static constexpr uint32_t kNumStartupFrames = 4; + +Awb::Awb() +{ +} + +int Awb::configure([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + /* + * Initially we have no idea what the colour balance will be like, so + * for the first frame we will make no assumptions and leave the R/B + * channels unmodified. + */ + context.activeState.awb.rGain = 1.0; + context.activeState.awb.bGain = 1.0; + + return 0; +} + +size_t Awb::fillGainsParamBlock(mali_c55_params_block block, IPAContext &context, + IPAFrameContext &frameContext) +{ + block.header->type = MALI_C55_PARAM_BLOCK_AWB_GAINS; + block.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE; + block.header->size = sizeof(struct mali_c55_params_awb_gains); + + double rGain = context.activeState.awb.rGain; + double bGain = context.activeState.awb.bGain; + + /* + * The gains here map as follows: + * gain00 = R + * gain01 = Gr + * gain10 = Gb + * gain11 = B + * + * This holds true regardless of the bayer order of the input data, as + * the mapping is done internally in the ISP. + */ + block.awb_gains->gain00 = floatingToFixedPoint<4, 8, uint16_t, double>(rGain); + block.awb_gains->gain01 = floatingToFixedPoint<4, 8, uint16_t, double>(1.0); + block.awb_gains->gain10 = floatingToFixedPoint<4, 8, uint16_t, double>(1.0); + block.awb_gains->gain11 = floatingToFixedPoint<4, 8, uint16_t, double>(bGain); + + frameContext.awb.rGain = rGain; + frameContext.awb.bGain = bGain; + + return sizeof(struct mali_c55_params_awb_gains); +} + +size_t Awb::fillConfigParamBlock(mali_c55_params_block block) +{ + block.header->type = MALI_C55_PARAM_BLOCK_AWB_CONFIG; + block.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE; + block.header->size = sizeof(struct mali_c55_params_awb_config); + + /* Tap the stats after the purple fringe block */ + block.awb_config->tap_point = MALI_C55_AWB_STATS_TAP_PF; + + /* Get R/G and B/G ratios as statistics */ + block.awb_config->stats_mode = MALI_C55_AWB_MODE_RGBG; + + /* Default white level */ + block.awb_config->white_level = 1023; + + /* Default black level */ + block.awb_config->black_level = 0; + + /* + * By default pixels are included who's colour ratios are bounded in a + * region (on a cr ratio x cb ratio graph) defined by four points: + * (0.25, 0.25) + * (0.25, 1.99609375) + * (1.99609375, 1.99609375) + * (1.99609375, 0.25) + * + * The ratios themselves are stored in Q4.8 format. + * + * \todo should these perhaps be tunable? + */ + block.awb_config->cr_max = 511; + block.awb_config->cr_min = 64; + block.awb_config->cb_max = 511; + block.awb_config->cb_min = 64; + + /* We use the full 15x15 zoning scheme */ + block.awb_config->nodes_used_horiz = 15; + block.awb_config->nodes_used_vert = 15; + + /* + * We set the trimming boundaries equivalent to the main boundaries. In + * other words; no trimming. + */ + block.awb_config->cr_high = 511; + block.awb_config->cr_low = 64; + block.awb_config->cb_high = 511; + block.awb_config->cb_low = 64; + + return sizeof(struct mali_c55_params_awb_config); +} + +void Awb::prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, mali_c55_params_buffer *params) +{ + mali_c55_params_block block; + block.data = ¶ms->data[params->total_size]; + + params->total_size += fillGainsParamBlock(block, context, frameContext); + + if (frame > 0) + return; + + block.data = ¶ms->data[params->total_size]; + params->total_size += fillConfigParamBlock(block); +} + +void Awb::process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, const mali_c55_stats_buffer *stats, + [[maybe_unused]] ControlList &metadata) +{ + const struct mali_c55_awb_average_ratios *awb_ratios = stats->awb_ratios; + + /* + * The ISP produces average R:G and B:G ratios for zones. We take the + * average of all the zones with data and simply invert them to provide + * gain figures that we can apply to approximate a grey world. + */ + unsigned int counted_zones = 0; + double rgSum = 0, bgSum = 0; + + for (unsigned int i = 0; i < 225; i++) { + if (!awb_ratios[i].num_pixels) + continue; + + /* + * The statistics are in Q4.8 format, so we convert to double + * here. + */ + rgSum += fixedToFloatingPoint<4, 8, double, uint16_t>(awb_ratios[i].avg_rg_gr); + bgSum += fixedToFloatingPoint<4, 8, double, uint16_t>(awb_ratios[i].avg_bg_br); + counted_zones++; + } + + /* + * Sometimes the first frame's statistics have no valid pixels, in which + * case we'll just assume a grey world until they say otherwise. + */ + double rgAvg, bgAvg; + if (!counted_zones) { + rgAvg = 1.0; + bgAvg = 1.0; + } else { + rgAvg = rgSum / counted_zones; + bgAvg = bgSum / counted_zones; + } + + /* + * The statistics are generated _after_ white balancing is performed in + * the ISP. To get the true ratio we therefore have to adjust the stats + * figure by the gains that were applied when the statistics for this + * frame were generated. + */ + double rRatio = rgAvg / frameContext.awb.rGain; + double bRatio = bgAvg / frameContext.awb.bGain; + + /* + * And then we can simply invert the ratio to find the gain we should + * apply. + */ + double rGain = 1 / rRatio; + double bGain = 1 / bRatio; + + /* + * Running at full speed, this algorithm results in oscillations in the + * colour balance. To remove those we dampen the speed at which it makes + * changes in gain, unless we're in the startup phase in which case we + * want to fix the miscolouring as quickly as possible. + */ + double speed = frame < kNumStartupFrames ? 1.0 : 0.2; + rGain = speed * rGain + context.activeState.awb.rGain * (1.0 - speed); + bGain = speed * bGain + context.activeState.awb.bGain * (1.0 - speed); + + context.activeState.awb.rGain = rGain; + context.activeState.awb.bGain = bGain; + + metadata.set(controls::ColourGains, { + static_cast<float>(frameContext.awb.rGain), + static_cast<float>(frameContext.awb.bGain), + }); + + LOG(MaliC55Awb, Debug) << "For frame number " << frame << ": " + << "Average R/G Ratio: " << rgAvg + << ", Average B/G Ratio: " << bgAvg + << "\nrGain applied to this frame: " << frameContext.awb.rGain + << ", bGain applied to this frame: " << frameContext.awb.bGain + << "\nrGain to apply: " << context.activeState.awb.rGain + << ", bGain to apply: " << context.activeState.awb.bGain; +} + +REGISTER_IPA_ALGORITHM(Awb, "Awb") + +} /* namespace ipa::mali_c55::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/awb.h b/src/ipa/mali-c55/algorithms/awb.h new file mode 100644 index 00000000..800c2e83 --- /dev/null +++ b/src/ipa/mali-c55/algorithms/awb.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas on Board Oy + * + * awb.h - Mali C55 grey world auto white balance algorithm + */ + +#include "algorithm.h" +#include "ipa_context.h" + +namespace libcamera { + +namespace ipa::mali_c55::algorithms { + +class Awb : public Algorithm +{ +public: + Awb(); + ~Awb() = default; + + int configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + mali_c55_params_buffer *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const mali_c55_stats_buffer *stats, + ControlList &metadata) override; + +private: + size_t fillGainsParamBlock(mali_c55_params_block block, + IPAContext &context, + IPAFrameContext &frameContext); + size_t fillConfigParamBlock(mali_c55_params_block block); +}; + +} /* namespace ipa::mali_c55::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/blc.cpp b/src/ipa/mali-c55/algorithms/blc.cpp new file mode 100644 index 00000000..2a54c86a --- /dev/null +++ b/src/ipa/mali-c55/algorithms/blc.cpp @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * Mali-C55 sensor offset (black level) correction + */ + +#include "blc.h" + +#include <libcamera/base/log.h> +#include <libcamera/control_ids.h> + +#include "libcamera/internal/yaml_parser.h" + +/** + * \file blc.h + */ + +namespace libcamera { + +namespace ipa::mali_c55::algorithms { + +/** + * \class BlackLevelCorrection + * \brief MaliC55 Black Level Correction control + */ + +LOG_DEFINE_CATEGORY(MaliC55Blc) + +BlackLevelCorrection::BlackLevelCorrection() + : tuningParameters_(false) +{ +} + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int BlackLevelCorrection::init([[maybe_unused]] IPAContext &context, + const YamlObject &tuningData) +{ + offset00 = tuningData["offset00"].get<uint32_t>(0); + offset01 = tuningData["offset01"].get<uint32_t>(0); + offset10 = tuningData["offset10"].get<uint32_t>(0); + offset11 = tuningData["offset11"].get<uint32_t>(0); + + if (offset00 > kMaxOffset || offset01 > kMaxOffset || + offset10 > kMaxOffset || offset11 > kMaxOffset) { + LOG(MaliC55Blc, Error) << "Invalid black level offsets"; + return -EINVAL; + } + + tuningParameters_ = true; + + LOG(MaliC55Blc, Debug) + << "Black levels: 00 " << offset00 << ", 01 " << offset01 + << ", 10 " << offset10 << ", 11 " << offset11; + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::configure + */ +int BlackLevelCorrection::configure(IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + /* + * If no Black Levels were passed in through tuning data then we could + * use the value from the CameraSensorHelper if one is available. + */ + if (context.configuration.sensor.blackLevel && + !(offset00 + offset01 + offset10 + offset11)) { + offset00 = context.configuration.sensor.blackLevel; + offset01 = context.configuration.sensor.blackLevel; + offset10 = context.configuration.sensor.blackLevel; + offset11 = context.configuration.sensor.blackLevel; + } + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void BlackLevelCorrection::prepare([[maybe_unused]] IPAContext &context, + const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + mali_c55_params_buffer *params) +{ + mali_c55_params_block block; + block.data = ¶ms->data[params->total_size]; + + if (frame > 0) + return; + + if (!tuningParameters_) + return; + + block.header->type = MALI_C55_PARAM_BLOCK_SENSOR_OFFS; + block.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE; + block.header->size = sizeof(mali_c55_params_sensor_off_preshading); + + block.sensor_offs->chan00 = offset00; + block.sensor_offs->chan01 = offset01; + block.sensor_offs->chan10 = offset10; + block.sensor_offs->chan11 = offset11; + + params->total_size += block.header->size; +} + +void BlackLevelCorrection::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const mali_c55_stats_buffer *stats, + ControlList &metadata) +{ + /* + * Black Level Offsets in tuning data need to be 20-bit, whereas the + * metadata expects values from a 16-bit range. Right-shift to remove + * the 4 least significant bits. + * + * The black levels should be reported in the order R, Gr, Gb, B. We + * ignore that here given we're using matching values so far, but it + * would be safer to check the sensor's bayer order. + * + * \todo Account for bayer order. + */ + metadata.set(controls::SensorBlackLevels, { + static_cast<int32_t>(offset00 >> 4), + static_cast<int32_t>(offset01 >> 4), + static_cast<int32_t>(offset10 >> 4), + static_cast<int32_t>(offset11 >> 4), + }); +} + +REGISTER_IPA_ALGORITHM(BlackLevelCorrection, "BlackLevelCorrection") + +} /* namespace ipa::mali_c55::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/blc.h b/src/ipa/mali-c55/algorithms/blc.h new file mode 100644 index 00000000..9696e8e9 --- /dev/null +++ b/src/ipa/mali-c55/algorithms/blc.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * Mali-C55 sensor offset (black level) correction + */ + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::mali_c55::algorithms { + +class BlackLevelCorrection : public Algorithm +{ +public: + BlackLevelCorrection(); + ~BlackLevelCorrection() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + mali_c55_params_buffer *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const mali_c55_stats_buffer *stats, + ControlList &metadata) override; + +private: + static constexpr uint32_t kMaxOffset = 0xfffff; + + bool tuningParameters_; + uint32_t offset00; + uint32_t offset01; + uint32_t offset10; + uint32_t offset11; +}; + +} /* namespace ipa::mali_c55::algorithms */ +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/lsc.cpp b/src/ipa/mali-c55/algorithms/lsc.cpp new file mode 100644 index 00000000..c5afc04d --- /dev/null +++ b/src/ipa/mali-c55/algorithms/lsc.cpp @@ -0,0 +1,216 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board Oy + * + * lsc.cpp - Mali-C55 Lens shading correction algorithm + */ + +#include "lsc.h" + +#include "libcamera/internal/yaml_parser.h" + +namespace libcamera { + +namespace ipa::mali_c55::algorithms { + +LOG_DEFINE_CATEGORY(MaliC55Lsc) + +int Lsc::init([[maybe_unused]] IPAContext &context, const YamlObject &tuningData) +{ + if (!tuningData.contains("meshScale")) { + LOG(MaliC55Lsc, Error) << "meshScale missing from tuningData"; + return -EINVAL; + } + + meshScale_ = tuningData["meshScale"].get<uint32_t>(0); + + const YamlObject &yamlSets = tuningData["sets"]; + if (!yamlSets.isList()) { + LOG(MaliC55Lsc, Error) << "LSC tables missing or invalid"; + return -EINVAL; + } + + size_t tableSize = 0; + const auto &sets = yamlSets.asList(); + for (const auto &yamlSet : sets) { + uint32_t ct = yamlSet["ct"].get<uint32_t>(0); + + if (!ct) { + LOG(MaliC55Lsc, Error) << "Invalid colour temperature"; + return -EINVAL; + } + + if (std::count(colourTemperatures_.begin(), + colourTemperatures_.end(), ct)) { + LOG(MaliC55Lsc, Error) + << "Multiple sets found for colour temperature"; + return -EINVAL; + } + + std::vector<uint8_t> rTable = + yamlSet["r"].getList<uint8_t>().value_or(std::vector<uint8_t>{}); + std::vector<uint8_t> gTable = + yamlSet["g"].getList<uint8_t>().value_or(std::vector<uint8_t>{}); + std::vector<uint8_t> bTable = + yamlSet["b"].getList<uint8_t>().value_or(std::vector<uint8_t>{}); + + /* + * Some validation to do; only 16x16 and 32x32 tables of + * coefficients are acceptable, and all tables across all of the + * sets must be the same size. The first time we encounter a + * table we check that it is an acceptable size and if so make + * sure all other tables are of equal size. + */ + if (!tableSize) { + if (rTable.size() != 256 && rTable.size() != 1024) { + LOG(MaliC55Lsc, Error) + << "Invalid table size for colour temperature " << ct; + return -EINVAL; + } + tableSize = rTable.size(); + } + + if (rTable.size() != tableSize || + gTable.size() != tableSize || + bTable.size() != tableSize) { + LOG(MaliC55Lsc, Error) + << "Invalid or mismatched table size for colour temperature " << ct; + return -EINVAL; + } + + if (colourTemperatures_.size() >= 3) { + LOG(MaliC55Lsc, Error) + << "A maximum of 3 colour temperatures are supported"; + return -EINVAL; + } + + for (unsigned int i = 0; i < tableSize; i++) { + mesh_[kRedOffset + i] |= + (rTable[i] << (colourTemperatures_.size() * 8)); + mesh_[kGreenOffset + i] |= + (gTable[i] << (colourTemperatures_.size() * 8)); + mesh_[kBlueOffset + i] |= + (bTable[i] << (colourTemperatures_.size() * 8)); + } + + colourTemperatures_.push_back(ct); + } + + /* + * The mesh has either 16x16 or 32x32 nodes, we tell the driver which it + * is based on the number of values in the tuning data's table. + */ + if (tableSize == 256) + meshSize_ = 15; + else + meshSize_ = 31; + + return 0; +} + +size_t Lsc::fillConfigParamsBlock(mali_c55_params_block block) const +{ + block.header->type = MALI_C55_PARAM_MESH_SHADING_CONFIG; + block.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE; + block.header->size = sizeof(struct mali_c55_params_mesh_shading_config); + + block.shading_config->mesh_show = false; + block.shading_config->mesh_scale = meshScale_; + block.shading_config->mesh_page_r = 0; + block.shading_config->mesh_page_g = 1; + block.shading_config->mesh_page_b = 2; + block.shading_config->mesh_width = meshSize_; + block.shading_config->mesh_height = meshSize_; + + std::copy(mesh_.begin(), mesh_.end(), block.shading_config->mesh); + + return block.header->size; +} + +size_t Lsc::fillSelectionParamsBlock(mali_c55_params_block block, uint8_t bank, + uint8_t alpha) const +{ + block.header->type = MALI_C55_PARAM_MESH_SHADING_SELECTION; + block.header->flags = MALI_C55_PARAM_BLOCK_FL_NONE; + block.header->size = sizeof(struct mali_c55_params_mesh_shading_selection); + + block.shading_selection->mesh_alpha_bank_r = bank; + block.shading_selection->mesh_alpha_bank_g = bank; + block.shading_selection->mesh_alpha_bank_b = bank; + block.shading_selection->mesh_alpha_r = alpha; + block.shading_selection->mesh_alpha_g = alpha; + block.shading_selection->mesh_alpha_b = alpha; + block.shading_selection->mesh_strength = 0x1000; /* Otherwise known as 1.0 */ + + return block.header->size; +} + +std::tuple<uint8_t, uint8_t> Lsc::findBankAndAlpha(uint32_t ct) const +{ + unsigned int i; + + ct = std::clamp<uint32_t>(ct, colourTemperatures_.front(), + colourTemperatures_.back()); + + for (i = 0; i < colourTemperatures_.size() - 1; i++) { + if (ct >= colourTemperatures_[i] && + ct <= colourTemperatures_[i + 1]) + break; + } + + /* + * With the clamping, we're guaranteed an index into colourTemperatures_ + * that's <= colourTemperatures_.size() - 1. + */ + uint8_t alpha = (255 * (ct - colourTemperatures_[i])) / + (colourTemperatures_[i + 1] - colourTemperatures_[i]); + + return { i, alpha }; +} + +void Lsc::prepare(IPAContext &context, [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + mali_c55_params_buffer *params) +{ + /* + * For each frame we assess the colour temperature of the **last** frame + * and then select an appropriately blended table of coefficients based + * on that ct. As a bit of a shortcut, if we've only a single table the + * handling is somewhat simpler; if it's the first frame we just select + * that table and if we're past the first frame then we can just do + * nothing - the config will never change. + */ + uint32_t temperatureK = context.activeState.agc.temperatureK; + uint8_t bank, alpha; + + if (colourTemperatures_.size() == 1) { + if (frame > 0) + return; + + bank = 0; + alpha = 0; + } else { + std::tie(bank, alpha) = findBankAndAlpha(temperatureK); + } + + mali_c55_params_block block; + block.data = ¶ms->data[params->total_size]; + + params->total_size += fillSelectionParamsBlock(block, bank, alpha); + + if (frame > 0) + return; + + /* + * If this is the first frame, we need to load the parsed coefficient + * tables from tuning data to the ISP. + */ + block.data = ¶ms->data[params->total_size]; + params->total_size += fillConfigParamsBlock(block); +} + +REGISTER_IPA_ALGORITHM(Lsc, "Lsc") + +} /* namespace ipa::mali_c55::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/lsc.h b/src/ipa/mali-c55/algorithms/lsc.h new file mode 100644 index 00000000..e613277a --- /dev/null +++ b/src/ipa/mali-c55/algorithms/lsc.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board Oy + * + * lsc.h - Mali-C55 Lens shading correction algorithm + */ + +#include <map> +#include <tuple> + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::mali_c55::algorithms { + +class Lsc : public Algorithm +{ +public: + Lsc() = default; + ~Lsc() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + mali_c55_params_buffer *params) override; +private: + static constexpr unsigned int kRedOffset = 0; + static constexpr unsigned int kGreenOffset = 1024; + static constexpr unsigned int kBlueOffset = 2048; + + size_t fillConfigParamsBlock(mali_c55_params_block block) const; + size_t fillSelectionParamsBlock(mali_c55_params_block block, + uint8_t bank, uint8_t alpha) const; + std::tuple<uint8_t, uint8_t> findBankAndAlpha(uint32_t ct) const; + + std::vector<uint32_t> mesh_ = std::vector<uint32_t>(3072); + std::vector<uint32_t> colourTemperatures_; + uint32_t meshScale_; + uint32_t meshSize_; +}; + +} /* namespace ipa::mali_c55::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/algorithms/meson.build b/src/ipa/mali-c55/algorithms/meson.build new file mode 100644 index 00000000..1665da07 --- /dev/null +++ b/src/ipa/mali-c55/algorithms/meson.build @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: CC0-1.0 + +mali_c55_ipa_algorithms = files([ + 'agc.cpp', + 'awb.cpp', + 'blc.cpp', + 'lsc.cpp', +]) diff --git a/src/ipa/mali-c55/data/imx415.yaml b/src/ipa/mali-c55/data/imx415.yaml new file mode 100644 index 00000000..126b427a --- /dev/null +++ b/src/ipa/mali-c55/data/imx415.yaml @@ -0,0 +1,325 @@ +# SPDX-License-Identifier: CC0-1.0 +%YAML 1.1 +--- +version: 1 +algorithms: + - Agc: + - Awb: + - BlackLevelCorrection: + offset00: 51200 + offset01: 51200 + offset10: 51200 + offset11: 51200 + - Lsc: + meshScale: 4 # 1.0 - 2.0 Gain + sets: + - ct: 2500 + r: [ + 21, 20, 19, 17, 15, 14, 12, 11, 9, 9, 9, 9, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 13, 13, 10, 10, 13, 16, 17, 18, 21, 22, + 21, 20, 18, 16, 14, 13, 12, 11, 10, 9, 9, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 12, 15, 17, 18, 21, 21, + 20, 19, 17, 16, 14, 13, 12, 11, 10, 9, 8, 8, 8, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 8, 8, 8, 11, 15, 17, 18, 21, 21, + 19, 19, 17, 15, 14, 13, 12, 11, 10, 8, 8, 7, 7, 7, 6, 6, 7, 7, 8, 8, 8, 8, 8, 7, 7, 8, 10, 14, 17, 18, 20, 22, + 19, 18, 17, 15, 14, 13, 11, 11, 9, 8, 8, 7, 7, 6, 5, 5, 5, 5, 6, 7, 8, 7, 7, 6, 7, 7, 10, 12, 16, 18, 20, 22, + 18, 18, 16, 15, 14, 12, 11, 10, 9, 8, 6, 6, 5, 5, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 12, 16, 18, 19, 20, + 18, 18, 16, 14, 13, 12, 11, 9, 9, 7, 6, 5, 5, 5, 4, 4, 4, 5, 5, 4, 4, 5, 5, 5, 5, 6, 8, 11, 15, 18, 18, 19, + 18, 17, 15, 14, 13, 12, 11, 9, 8, 7, 6, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 6, 7, 9, 14, 17, 18, 18, + 18, 17, 15, 14, 13, 12, 11, 9, 8, 7, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 6, 8, 12, 17, 18, 18, + 18, 16, 15, 13, 12, 11, 10, 9, 8, 7, 5, 4, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 5, 6, 8, 12, 16, 19, 19, + 17, 16, 15, 13, 12, 11, 10, 8, 7, 6, 4, 4, 3, 3, 3, 3, 3, 3, 4, 4, 3, 3, 3, 4, 4, 5, 6, 9, 12, 16, 19, 20, + 17, 15, 15, 13, 12, 11, 10, 8, 6, 6, 5, 4, 3, 3, 3, 2, 3, 3, 4, 4, 3, 3, 2, 3, 4, 5, 6, 9, 11, 16, 19, 20, + 17, 15, 15, 14, 11, 11, 10, 8, 6, 5, 5, 4, 3, 3, 2, 2, 2, 3, 3, 3, 3, 3, 2, 3, 4, 5, 6, 8, 11, 16, 18, 19, + 16, 16, 15, 13, 11, 11, 10, 7, 6, 5, 4, 4, 3, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 3, 4, 6, 8, 11, 14, 17, 18, + 16, 16, 14, 13, 11, 10, 9, 7, 6, 5, 4, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 3, 4, 6, 8, 10, 14, 17, 18, + 16, 15, 14, 13, 13, 10, 9, 7, 6, 4, 4, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 4, 4, 6, 8, 11, 17, 18, 19, + 16, 15, 14, 14, 13, 12, 9, 8, 7, 5, 4, 2, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 4, 5, 6, 8, 12, 17, 19, 20, + 17, 15, 15, 14, 13, 12, 9, 8, 7, 5, 3, 2, 1, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5, 6, 8, 13, 16, 19, 21, + 17, 16, 15, 13, 13, 12, 9, 8, 7, 5, 3, 2, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 5, 5, 7, 8, 13, 16, 19, 20, + 17, 16, 15, 14, 13, 12, 9, 8, 7, 5, 3, 2, 0, 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 5, 6, 7, 9, 13, 17, 19, 20, + 18, 16, 15, 14, 13, 12, 9, 8, 7, 5, 4, 2, 1, 0, 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 5, 6, 8, 9, 13, 17, 20, 20, + 18, 16, 16, 15, 14, 12, 10, 9, 7, 6, 5, 3, 2, 1, 0, 0, 0, 1, 2, 3, 3, 3, 4, 5, 6, 7, 9, 10, 14, 18, 20, 20, + 18, 17, 16, 15, 14, 12, 10, 9, 8, 7, 6, 5, 3, 3, 1, 0, 0, 1, 2, 3, 4, 4, 5, 6, 6, 7, 9, 12, 15, 19, 20, 20, + 18, 18, 17, 16, 14, 13, 11, 10, 9, 8, 7, 6, 5, 5, 3, 1, 1, 1, 2, 3, 5, 5, 5, 6, 7, 9, 12, 15, 17, 19, 20, 20, + 18, 18, 17, 16, 15, 13, 12, 10, 10, 9, 8, 7, 6, 5, 4, 2, 1, 2, 3, 4, 5, 5, 6, 6, 8, 10, 13, 16, 18, 20, 20, 21, + 19, 18, 17, 16, 15, 14, 13, 11, 10, 10, 9, 8, 7, 6, 5, 4, 3, 2, 3, 3, 5, 5, 6, 7, 10, 11, 14, 17, 19, 20, 21, 22, + 20, 19, 18, 17, 16, 15, 13, 12, 11, 10, 10, 9, 8, 7, 6, 5, 4, 3, 3, 3, 5, 6, 6, 7, 10, 12, 14, 18, 20, 21, 22, 23, + 21, 20, 19, 18, 17, 16, 14, 13, 12, 11, 10, 10, 9, 7, 7, 5, 5, 4, 4, 5, 6, 6, 7, 8, 11, 13, 16, 19, 21, 22, 22, 22, + 22, 21, 20, 19, 18, 17, 16, 14, 13, 12, 12, 10, 9, 8, 7, 6, 6, 5, 5, 6, 7, 7, 8, 9, 12, 14, 18, 20, 21, 22, 22, 22, + 23, 22, 21, 20, 19, 17, 16, 15, 14, 14, 13, 12, 10, 9, 8, 7, 6, 5, 5, 6, 7, 8, 8, 10, 12, 15, 18, 20, 21, 22, 22, 22, + 24, 23, 22, 21, 20, 18, 17, 16, 15, 15, 14, 14, 13, 11, 9, 8, 6, 6, 6, 6, 7, 8, 9, 11, 14, 17, 19, 20, 21, 21, 21, 21, + 24, 24, 23, 21, 20, 19, 17, 16, 15, 15, 15, 14, 14, 14, 11, 9, 6, 5, 5, 6, 8, 8, 10, 12, 15, 17, 20, 20, 21, 21, 21, 21, + ] + g: [ + 19, 18, 17, 15, 13, 12, 10, 9, 8, 8, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 12, 12, 9, 9, 11, 15, 15, 16, 19, 20, + 19, 18, 16, 15, 12, 12, 10, 10, 8, 7, 7, 7, 7, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 7, 7, 11, 14, 15, 16, 19, 19, + 18, 17, 16, 14, 12, 12, 10, 10, 8, 7, 7, 6, 6, 5, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 10, 14, 15, 17, 19, 19, + 17, 17, 16, 14, 12, 12, 10, 10, 8, 7, 6, 6, 6, 5, 5, 4, 5, 5, 6, 6, 6, 7, 7, 5, 6, 6, 9, 13, 15, 17, 18, 20, + 17, 17, 15, 14, 12, 11, 10, 9, 8, 7, 6, 5, 5, 4, 4, 4, 4, 4, 5, 6, 6, 6, 5, 5, 5, 6, 8, 11, 15, 17, 18, 20, + 17, 17, 15, 13, 12, 11, 9, 9, 8, 7, 5, 5, 4, 4, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 7, 11, 15, 17, 18, 18, + 17, 16, 15, 13, 12, 11, 9, 8, 8, 6, 5, 4, 4, 4, 3, 3, 4, 4, 3, 3, 3, 4, 4, 4, 4, 5, 6, 9, 14, 17, 17, 18, + 17, 16, 14, 13, 12, 11, 9, 8, 7, 6, 5, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 6, 8, 13, 16, 17, 17, + 17, 15, 14, 13, 12, 11, 9, 8, 7, 6, 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 7, 12, 16, 17, 17, + 17, 15, 14, 12, 11, 10, 9, 8, 7, 6, 4, 4, 3, 3, 3, 2, 2, 3, 3, 3, 2, 2, 3, 3, 3, 4, 5, 7, 11, 15, 18, 18, + 16, 14, 13, 12, 11, 10, 9, 7, 7, 5, 4, 3, 3, 3, 2, 2, 2, 3, 3, 3, 2, 2, 2, 3, 3, 4, 5, 8, 11, 15, 18, 19, + 16, 14, 13, 12, 11, 10, 9, 7, 5, 5, 4, 3, 3, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 3, 4, 5, 8, 10, 15, 18, 19, + 16, 14, 14, 13, 11, 10, 9, 7, 5, 5, 4, 3, 2, 2, 2, 2, 2, 2, 3, 3, 2, 2, 2, 2, 3, 3, 5, 7, 10, 15, 17, 18, + 16, 15, 14, 12, 11, 10, 9, 7, 5, 5, 4, 3, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 3, 5, 6, 10, 14, 17, 17, + 15, 15, 13, 12, 11, 10, 9, 7, 5, 4, 3, 2, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 1, 2, 2, 3, 5, 7, 10, 14, 17, 18, + 15, 14, 13, 12, 12, 10, 9, 7, 6, 4, 3, 2, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 4, 5, 7, 10, 17, 18, 18, + 15, 14, 14, 13, 12, 11, 9, 7, 6, 4, 3, 2, 1, 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 5, 7, 12, 17, 19, 19, + 16, 14, 14, 13, 12, 12, 9, 7, 6, 4, 3, 2, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4, 4, 5, 8, 12, 17, 19, 20, + 16, 15, 14, 13, 12, 12, 9, 7, 7, 4, 3, 1, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 6, 8, 12, 17, 19, 20, + 17, 15, 14, 13, 12, 12, 9, 7, 7, 5, 3, 1, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6, 8, 12, 17, 19, 20, + 18, 15, 15, 14, 13, 12, 9, 8, 7, 5, 4, 2, 1, 0, 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 5, 6, 7, 9, 13, 17, 19, 20, + 18, 16, 15, 14, 13, 12, 9, 9, 7, 6, 5, 3, 2, 1, 0, 0, 0, 1, 2, 3, 3, 3, 4, 5, 6, 7, 9, 10, 14, 18, 20, 20, + 18, 16, 16, 15, 13, 12, 10, 9, 8, 7, 6, 5, 3, 3, 1, 0, 0, 1, 2, 3, 4, 4, 5, 5, 6, 7, 9, 12, 15, 19, 20, 20, + 18, 18, 16, 16, 14, 13, 10, 10, 9, 8, 7, 6, 5, 5, 3, 1, 1, 1, 2, 3, 5, 5, 5, 6, 6, 8, 11, 15, 17, 19, 20, 20, + 18, 18, 16, 16, 14, 13, 12, 10, 9, 9, 8, 7, 6, 5, 4, 3, 1, 2, 3, 5, 5, 5, 5, 6, 7, 10, 12, 15, 18, 20, 20, 20, + 18, 18, 17, 16, 15, 14, 12, 11, 10, 10, 9, 8, 7, 6, 5, 4, 3, 2, 3, 3, 5, 5, 5, 6, 9, 11, 14, 16, 19, 20, 20, 21, + 19, 19, 18, 17, 16, 15, 13, 12, 11, 10, 10, 9, 8, 7, 5, 4, 4, 3, 3, 3, 5, 5, 6, 7, 10, 12, 14, 18, 20, 20, 21, 22, + 21, 20, 18, 18, 17, 16, 14, 12, 11, 11, 10, 10, 8, 7, 7, 5, 4, 4, 4, 5, 6, 6, 6, 7, 11, 13, 16, 19, 20, 21, 21, 22, + 22, 21, 20, 19, 18, 16, 15, 14, 12, 12, 12, 10, 9, 8, 7, 6, 5, 4, 5, 6, 6, 7, 7, 8, 12, 13, 17, 19, 21, 21, 21, 21, + 23, 22, 21, 20, 18, 17, 16, 16, 14, 14, 13, 12, 10, 10, 8, 7, 6, 5, 5, 6, 7, 7, 8, 9, 12, 15, 18, 19, 21, 21, 21, 20, + 23, 22, 22, 21, 20, 18, 17, 16, 15, 15, 15, 14, 13, 11, 9, 8, 6, 6, 6, 6, 7, 8, 9, 10, 13, 16, 19, 20, 20, 21, 21, 20, + 24, 23, 22, 21, 20, 19, 17, 17, 16, 16, 15, 15, 14, 14, 11, 9, 6, 6, 6, 6, 8, 8, 10, 12, 15, 17, 19, 20, 20, 21, 21, 20, + ] + b: [ + 11, 9, 9, 7, 6, 5, 4, 4, 3, 3, 3, 3, 3, 4, 5, 5, 5, 4, 5, 4, 4, 4, 7, 7, 3, 3, 5, 8, 8, 9, 11, 11, + 11, 10, 8, 7, 5, 5, 5, 4, 3, 3, 3, 3, 3, 3, 4, 4, 5, 4, 5, 4, 4, 4, 4, 4, 2, 2, 5, 8, 8, 9, 11, 11, + 10, 10, 7, 7, 5, 5, 5, 4, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 3, 1, 1, 5, 7, 9, 9, 11, 11, + 10, 9, 8, 7, 6, 5, 5, 5, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 3, 2, 1, 1, 4, 7, 9, 10, 12, 13, + 9, 9, 8, 7, 6, 5, 5, 5, 4, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 3, 4, 3, 2, 1, 1, 1, 4, 6, 9, 10, 12, 13, + 9, 9, 9, 7, 6, 6, 5, 4, 4, 3, 3, 3, 2, 2, 2, 2, 3, 3, 2, 1, 2, 2, 2, 1, 1, 1, 2, 6, 9, 10, 11, 12, + 8, 9, 9, 7, 7, 6, 5, 4, 4, 3, 3, 2, 2, 2, 2, 2, 3, 3, 2, 1, 1, 1, 1, 0, 0, 1, 2, 5, 9, 11, 11, 11, + 8, 9, 9, 7, 7, 6, 5, 4, 4, 3, 3, 2, 2, 2, 2, 2, 3, 3, 2, 1, 1, 1, 1, 0, 0, 0, 1, 3, 8, 11, 11, 11, + 9, 9, 8, 7, 7, 7, 6, 4, 4, 3, 3, 2, 2, 2, 2, 2, 3, 3, 2, 1, 1, 1, 1, 0, 0, 0, 1, 3, 7, 11, 11, 11, + 9, 9, 8, 7, 7, 7, 6, 5, 4, 3, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 0, 0, 0, 0, 0, 1, 3, 7, 10, 12, 13, + 9, 9, 8, 8, 7, 7, 6, 5, 4, 3, 2, 1, 2, 1, 2, 2, 2, 3, 2, 1, 0, 0, 0, 0, 0, 0, 1, 4, 7, 10, 13, 13, + 9, 8, 8, 8, 7, 7, 6, 5, 3, 3, 2, 1, 1, 1, 1, 1, 2, 3, 3, 2, 1, 0, 0, 0, 0, 1, 2, 4, 7, 11, 13, 13, + 9, 8, 8, 7, 7, 6, 6, 5, 3, 3, 2, 2, 2, 1, 1, 1, 2, 3, 3, 2, 1, 0, 0, 0, 0, 1, 2, 4, 6, 11, 13, 13, + 9, 8, 8, 7, 7, 6, 6, 4, 3, 3, 2, 2, 1, 1, 1, 1, 2, 3, 3, 3, 1, 1, 0, 0, 1, 1, 2, 3, 6, 10, 12, 13, + 9, 8, 8, 7, 7, 6, 6, 4, 3, 3, 2, 1, 1, 1, 1, 1, 1, 3, 3, 3, 1, 1, 1, 1, 1, 2, 2, 4, 6, 10, 13, 13, + 9, 9, 8, 8, 8, 6, 6, 4, 4, 3, 2, 1, 1, 1, 1, 1, 2, 2, 3, 3, 1, 1, 1, 1, 2, 2, 3, 4, 6, 13, 14, 14, + 9, 9, 8, 8, 8, 8, 6, 5, 4, 3, 2, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 3, 3, 5, 9, 13, 15, 15, + 10, 9, 9, 9, 10, 10, 6, 6, 5, 3, 2, 1, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 4, 5, 9, 13, 15, 16, + 10, 10, 9, 9, 10, 10, 7, 6, 5, 3, 2, 1, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 4, 5, 9, 13, 15, 16, + 11, 10, 9, 9, 10, 10, 7, 6, 6, 3, 2, 1, 0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 5, 6, 9, 13, 15, 16, + 12, 10, 10, 10, 10, 10, 7, 6, 6, 4, 3, 1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 3, 5, 5, 6, 7, 10, 14, 15, 16, + 12, 11, 10, 10, 10, 10, 8, 7, 6, 6, 4, 3, 2, 1, 1, 1, 1, 2, 2, 3, 4, 4, 4, 4, 5, 5, 6, 8, 11, 15, 16, 16, + 12, 12, 11, 11, 10, 10, 9, 8, 7, 6, 6, 4, 4, 3, 2, 2, 2, 2, 3, 4, 4, 4, 5, 5, 5, 6, 7, 10, 13, 15, 16, 15, + 12, 12, 12, 12, 11, 10, 10, 8, 8, 7, 7, 6, 5, 5, 3, 2, 2, 2, 3, 4, 5, 5, 5, 5, 6, 7, 10, 13, 14, 16, 16, 16, + 12, 12, 13, 12, 12, 11, 11, 9, 8, 8, 8, 7, 6, 6, 5, 4, 3, 3, 3, 5, 6, 6, 6, 6, 7, 9, 11, 14, 15, 17, 17, 16, + 13, 13, 13, 13, 12, 12, 11, 11, 10, 10, 9, 8, 7, 7, 6, 5, 5, 4, 4, 4, 5, 6, 6, 6, 9, 10, 12, 14, 16, 17, 17, 18, + 13, 13, 14, 13, 13, 13, 12, 12, 11, 10, 10, 9, 8, 7, 6, 6, 6, 5, 4, 4, 6, 6, 6, 7, 9, 11, 12, 16, 17, 17, 17, 18, + 15, 15, 15, 15, 14, 13, 13, 13, 12, 11, 11, 10, 9, 9, 8, 7, 6, 5, 5, 5, 6, 7, 7, 8, 10, 12, 14, 17, 17, 18, 17, 17, + 16, 16, 16, 16, 15, 14, 14, 14, 13, 13, 12, 11, 11, 9, 8, 7, 7, 6, 6, 6, 7, 7, 8, 8, 10, 12, 16, 17, 18, 18, 17, 17, + 18, 17, 17, 16, 16, 15, 14, 14, 14, 14, 14, 13, 11, 11, 9, 8, 7, 7, 6, 6, 7, 7, 8, 9, 10, 13, 16, 17, 17, 18, 17, 16, + 18, 17, 17, 17, 16, 15, 15, 15, 15, 15, 15, 14, 14, 12, 10, 9, 7, 7, 6, 6, 7, 7, 8, 9, 11, 14, 16, 16, 17, 17, 16, 15, + 18, 18, 17, 17, 17, 16, 15, 15, 15, 16, 15, 15, 14, 14, 12, 9, 7, 6, 6, 6, 7, 7, 9, 10, 12, 14, 15, 16, 16, 16, 15, 15, + ] + - ct: 5500 + r: [ + 19, 18, 17, 16, 15, 13, 11, 10, 9, 9, 9, 8, 8, 8, 8, 8, 8, 9, 10, 10, 8, 8, 9, 9, 9, 11, 14, 15, 16, 16, 18, 18, + 18, 18, 17, 15, 14, 13, 11, 11, 9, 9, 8, 8, 7, 7, 7, 7, 7, 8, 9, 8, 8, 8, 9, 9, 8, 8, 11, 14, 16, 17, 17, 18, + 18, 17, 17, 15, 14, 13, 12, 11, 9, 9, 8, 7, 7, 6, 6, 6, 6, 8, 8, 8, 8, 8, 8, 8, 7, 7, 10, 13, 16, 17, 18, 18, + 17, 17, 16, 15, 14, 12, 11, 11, 9, 8, 7, 7, 6, 5, 5, 5, 5, 6, 8, 7, 8, 8, 8, 6, 6, 6, 9, 12, 16, 16, 18, 19, + 17, 17, 16, 14, 13, 12, 11, 10, 10, 8, 7, 7, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 6, 6, 6, 6, 8, 11, 15, 16, 17, 19, + 18, 17, 16, 14, 13, 12, 11, 10, 10, 8, 7, 6, 5, 5, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 10, 14, 16, 17, 18, + 18, 17, 16, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 4, 4, 4, 4, 4, 3, 4, 5, 5, 5, 5, 5, 6, 10, 13, 16, 16, 17, + 18, 16, 15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 4, 4, 4, 4, 4, 3, 3, 3, 4, 4, 4, 4, 5, 6, 8, 12, 16, 16, 16, + 17, 16, 15, 13, 12, 11, 10, 9, 8, 7, 5, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 4, 4, 5, 7, 11, 16, 16, 16, + 17, 16, 15, 12, 12, 12, 10, 8, 7, 7, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 4, 5, 7, 10, 16, 16, 16, + 16, 16, 14, 12, 12, 11, 10, 8, 7, 6, 4, 4, 3, 3, 3, 3, 3, 4, 3, 3, 2, 2, 2, 3, 3, 4, 4, 7, 10, 14, 16, 16, + 16, 15, 14, 13, 12, 11, 10, 8, 7, 6, 5, 4, 3, 3, 3, 3, 3, 4, 3, 3, 3, 2, 2, 3, 3, 3, 5, 7, 10, 14, 15, 16, + 16, 15, 15, 13, 11, 11, 11, 9, 7, 5, 5, 4, 3, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 3, 3, 5, 6, 9, 14, 15, 16, + 16, 15, 15, 13, 11, 11, 11, 10, 7, 5, 4, 4, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 3, 3, 5, 6, 10, 13, 15, 17, + 15, 15, 14, 12, 11, 11, 11, 10, 7, 4, 4, 3, 2, 2, 2, 2, 2, 2, 3, 3, 2, 2, 2, 3, 3, 3, 5, 6, 9, 13, 16, 17, + 15, 15, 14, 12, 11, 11, 10, 9, 7, 4, 4, 3, 1, 1, 1, 1, 2, 2, 3, 3, 2, 2, 3, 3, 3, 3, 5, 7, 10, 15, 17, 17, + 15, 15, 14, 12, 11, 11, 10, 9, 7, 4, 4, 3, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 5, 7, 11, 16, 17, 18, + 16, 15, 15, 12, 12, 11, 10, 9, 6, 4, 4, 3, 1, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 7, 12, 15, 17, 18, + 16, 16, 15, 12, 12, 11, 10, 8, 6, 5, 4, 3, 1, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 5, 6, 8, 12, 15, 17, 18, + 15, 15, 15, 13, 12, 12, 10, 8, 7, 5, 4, 3, 2, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 5, 7, 9, 12, 16, 17, 18, + 15, 15, 15, 13, 13, 12, 10, 8, 7, 5, 5, 3, 2, 1, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 5, 5, 7, 9, 13, 16, 18, 18, + 15, 15, 15, 14, 13, 12, 10, 8, 7, 6, 5, 5, 3, 2, 1, 0, 1, 1, 1, 2, 3, 3, 3, 5, 5, 6, 8, 10, 14, 17, 18, 19, + 16, 16, 16, 15, 13, 12, 10, 9, 8, 7, 6, 6, 5, 4, 2, 1, 1, 1, 1, 2, 3, 3, 5, 5, 6, 7, 9, 12, 15, 18, 18, 19, + 17, 16, 16, 16, 13, 12, 11, 10, 9, 8, 7, 7, 6, 5, 4, 2, 1, 1, 2, 3, 4, 4, 5, 5, 6, 8, 11, 14, 16, 18, 19, 19, + 18, 18, 17, 16, 14, 13, 12, 10, 9, 8, 8, 7, 7, 5, 5, 3, 2, 2, 4, 4, 5, 5, 5, 5, 7, 10, 12, 16, 17, 19, 19, 20, + 18, 18, 17, 16, 15, 13, 12, 10, 10, 9, 9, 9, 7, 6, 5, 4, 3, 3, 4, 4, 5, 5, 5, 6, 7, 11, 15, 17, 18, 19, 19, 20, + 19, 18, 18, 17, 16, 14, 13, 11, 10, 10, 9, 9, 8, 6, 5, 5, 4, 4, 4, 4, 6, 6, 6, 7, 9, 11, 15, 17, 19, 19, 20, 21, + 20, 19, 19, 19, 17, 16, 13, 12, 11, 11, 10, 9, 9, 7, 6, 5, 5, 4, 4, 5, 6, 7, 7, 8, 9, 12, 15, 18, 19, 19, 20, 20, + 21, 20, 20, 19, 19, 16, 16, 13, 12, 12, 11, 11, 9, 8, 7, 6, 6, 5, 5, 6, 7, 7, 8, 8, 10, 13, 17, 19, 19, 20, 20, 20, + 22, 21, 20, 20, 19, 17, 16, 14, 13, 13, 14, 12, 11, 9, 8, 7, 6, 5, 5, 6, 7, 7, 8, 9, 11, 15, 17, 19, 19, 20, 20, 20, + 22, 22, 21, 20, 19, 18, 16, 15, 14, 14, 15, 15, 13, 11, 9, 8, 7, 5, 5, 6, 7, 8, 9, 10, 13, 16, 18, 18, 19, 20, 19, 19, + 22, 22, 21, 20, 19, 19, 16, 16, 15, 15, 15, 15, 15, 13, 10, 9, 7, 5, 5, 6, 8, 8, 10, 11, 14, 16, 18, 18, 19, 19, 19, 19, + ] + g: [ + 16, 16, 15, 14, 13, 11, 10, 9, 8, 7, 7, 7, 6, 6, 6, 6, 6, 7, 8, 8, 6, 6, 7, 7, 7, 9, 12, 13, 13, 14, 14, 14, + 16, 16, 15, 14, 13, 11, 10, 9, 8, 7, 7, 6, 6, 6, 6, 6, 6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 10, 12, 13, 14, 14, 14, + 16, 15, 15, 13, 13, 11, 10, 9, 8, 7, 7, 6, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 5, 5, 8, 12, 14, 15, 15, 16, + 15, 15, 14, 13, 12, 11, 10, 10, 8, 7, 6, 6, 5, 4, 4, 4, 4, 5, 6, 6, 6, 6, 6, 5, 5, 5, 7, 11, 14, 15, 15, 17, + 16, 15, 14, 13, 12, 11, 10, 10, 9, 7, 6, 6, 4, 4, 4, 3, 3, 4, 4, 6, 6, 5, 5, 4, 4, 5, 6, 10, 14, 14, 16, 16, + 16, 15, 15, 13, 12, 11, 10, 10, 9, 7, 6, 6, 4, 4, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 9, 13, 14, 15, 16, + 16, 15, 15, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 5, 8, 13, 14, 15, 15, + 16, 15, 14, 12, 11, 10, 9, 8, 7, 6, 6, 5, 4, 3, 3, 3, 3, 3, 2, 2, 2, 3, 3, 3, 3, 4, 5, 7, 11, 14, 15, 15, + 16, 15, 14, 12, 11, 10, 9, 8, 7, 6, 5, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 3, 3, 4, 6, 10, 14, 15, 14, + 16, 15, 14, 12, 11, 11, 9, 8, 7, 6, 4, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 4, 6, 10, 15, 15, 15, + 15, 15, 13, 12, 11, 11, 10, 8, 7, 5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 6, 9, 13, 15, 15, + 15, 14, 13, 12, 11, 11, 10, 8, 7, 5, 5, 3, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 4, 6, 9, 13, 14, 15, + 15, 14, 14, 13, 11, 11, 10, 8, 7, 5, 5, 3, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 1, 2, 2, 2, 4, 5, 8, 13, 14, 15, + 15, 14, 14, 13, 11, 11, 11, 10, 7, 5, 4, 3, 2, 2, 2, 2, 2, 2, 3, 3, 2, 2, 1, 2, 2, 2, 4, 5, 8, 13, 14, 15, + 15, 14, 13, 12, 11, 11, 10, 9, 7, 4, 4, 3, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 5, 8, 13, 15, 16, + 15, 15, 13, 12, 11, 11, 10, 9, 7, 4, 4, 2, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 4, 6, 8, 15, 16, 16, + 15, 15, 14, 12, 11, 11, 10, 9, 7, 4, 4, 2, 1, 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 6, 11, 15, 16, 17, + 15, 15, 15, 12, 12, 11, 10, 10, 7, 4, 4, 2, 1, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 6, 11, 15, 17, 17, + 15, 15, 15, 12, 12, 12, 10, 9, 7, 5, 4, 2, 1, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 5, 7, 11, 15, 17, 17, + 15, 15, 15, 13, 12, 12, 10, 9, 7, 5, 4, 3, 2, 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 3, 4, 4, 6, 8, 12, 15, 17, 17, + 15, 15, 15, 13, 13, 12, 10, 9, 7, 6, 5, 3, 2, 1, 0, 0, 1, 1, 1, 2, 3, 3, 3, 3, 4, 5, 6, 9, 13, 16, 17, 18, + 15, 15, 15, 14, 13, 13, 10, 9, 8, 6, 6, 5, 3, 2, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 6, 7, 10, 14, 17, 18, 18, + 15, 16, 16, 15, 13, 13, 11, 9, 8, 7, 6, 6, 5, 4, 2, 1, 1, 1, 2, 2, 3, 4, 5, 5, 6, 7, 8, 12, 15, 17, 18, 18, + 16, 16, 16, 16, 13, 13, 11, 10, 9, 8, 7, 7, 6, 6, 4, 2, 2, 2, 2, 3, 5, 5, 5, 5, 6, 8, 11, 14, 16, 18, 18, 18, + 17, 17, 17, 16, 14, 13, 13, 11, 10, 9, 8, 8, 7, 6, 5, 4, 2, 2, 4, 5, 5, 5, 5, 5, 7, 9, 12, 15, 17, 18, 18, 18, + 18, 18, 17, 16, 15, 14, 13, 11, 10, 10, 9, 9, 7, 6, 5, 5, 4, 3, 4, 4, 5, 5, 5, 6, 7, 10, 15, 16, 18, 18, 18, 19, + 19, 18, 18, 17, 16, 14, 13, 12, 11, 10, 10, 9, 9, 7, 6, 5, 5, 4, 4, 4, 6, 6, 6, 6, 8, 10, 15, 16, 18, 18, 18, 19, + 20, 19, 19, 19, 17, 16, 14, 13, 12, 11, 11, 10, 9, 7, 7, 6, 5, 4, 4, 5, 6, 6, 6, 7, 9, 11, 15, 17, 18, 18, 18, 18, + 22, 20, 20, 20, 19, 17, 16, 14, 13, 13, 12, 11, 10, 9, 7, 6, 6, 5, 5, 6, 7, 7, 7, 8, 9, 12, 16, 18, 18, 19, 18, 18, + 22, 22, 21, 20, 19, 18, 17, 16, 14, 14, 15, 13, 11, 10, 9, 8, 7, 6, 5, 6, 7, 8, 8, 9, 10, 14, 17, 18, 18, 19, 18, 17, + 22, 22, 22, 21, 20, 19, 17, 17, 16, 16, 16, 16, 14, 12, 10, 8, 7, 6, 6, 7, 8, 8, 8, 10, 13, 16, 18, 18, 18, 18, 18, 17, + 22, 22, 22, 21, 20, 20, 18, 17, 16, 16, 17, 17, 16, 14, 11, 10, 8, 6, 6, 7, 8, 8, 10, 11, 14, 17, 18, 18, 18, 18, 18, 17, + ] + b: [ + 13, 12, 12, 12, 11, 9, 8, 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 6, 7, 7, 6, 6, 5, 5, 5, 6, 9, 9, 9, 10, 9, 8, + 13, 13, 12, 11, 11, 9, 9, 8, 7, 7, 7, 6, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 5, 5, 4, 4, 7, 9, 9, 10, 10, 9, + 13, 13, 12, 11, 11, 9, 9, 8, 7, 7, 6, 6, 5, 4, 4, 4, 4, 5, 6, 6, 6, 5, 5, 5, 3, 3, 6, 9, 10, 11, 10, 11, + 13, 13, 12, 11, 11, 10, 9, 9, 7, 7, 6, 5, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 4, 3, 3, 5, 8, 10, 11, 11, 12, + 13, 13, 12, 11, 11, 10, 9, 9, 8, 7, 6, 6, 4, 4, 3, 3, 3, 4, 4, 5, 5, 5, 4, 3, 2, 3, 4, 8, 11, 11, 11, 12, + 13, 13, 13, 11, 11, 10, 9, 9, 8, 7, 6, 6, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 3, 7, 11, 11, 11, 12, + 14, 14, 13, 11, 11, 10, 9, 8, 8, 7, 6, 6, 4, 3, 3, 3, 3, 3, 2, 2, 2, 3, 2, 2, 2, 2, 3, 6, 11, 11, 11, 11, + 14, 14, 13, 11, 11, 10, 9, 8, 7, 7, 6, 5, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 9, 11, 11, 11, + 14, 14, 13, 12, 11, 11, 9, 8, 7, 6, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 8, 12, 12, 11, + 14, 13, 13, 12, 12, 11, 10, 8, 7, 6, 4, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 4, 8, 12, 12, 13, + 13, 13, 13, 12, 12, 12, 11, 9, 7, 6, 4, 3, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 4, 8, 12, 12, 13, + 14, 13, 13, 12, 12, 11, 11, 9, 7, 6, 5, 3, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 4, 8, 12, 13, 13, + 14, 13, 13, 13, 12, 11, 11, 9, 7, 6, 5, 4, 2, 1, 1, 1, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 4, 7, 12, 13, 13, + 14, 13, 13, 13, 12, 12, 12, 11, 7, 5, 5, 4, 2, 1, 1, 1, 1, 3, 3, 3, 2, 2, 2, 2, 2, 3, 3, 4, 7, 12, 13, 13, + 14, 14, 13, 13, 12, 12, 11, 11, 7, 5, 4, 3, 1, 1, 1, 1, 1, 2, 3, 3, 2, 2, 2, 3, 3, 3, 4, 4, 7, 12, 13, 14, + 14, 14, 13, 13, 12, 12, 11, 11, 8, 5, 4, 3, 1, 1, 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 8, 13, 15, 15, + 14, 15, 14, 13, 13, 12, 11, 11, 8, 5, 4, 3, 1, 0, 0, 0, 0, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 10, 15, 15, 16, + 15, 15, 15, 14, 13, 13, 12, 11, 8, 5, 4, 3, 1, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 6, 11, 15, 16, 16, + 15, 15, 15, 14, 14, 14, 12, 11, 8, 6, 5, 3, 1, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 6, 7, 11, 15, 16, 16, + 15, 15, 15, 15, 14, 14, 12, 11, 9, 7, 5, 3, 2, 1, 0, 1, 1, 1, 2, 3, 3, 4, 4, 4, 4, 5, 6, 8, 12, 15, 16, 16, + 15, 16, 15, 15, 15, 14, 13, 11, 9, 7, 6, 5, 3, 2, 1, 1, 1, 1, 2, 3, 3, 4, 4, 4, 5, 6, 7, 9, 12, 16, 16, 16, + 15, 16, 16, 15, 15, 15, 13, 11, 10, 8, 7, 6, 4, 3, 2, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 7, 8, 10, 14, 16, 16, 16, + 16, 16, 17, 16, 15, 15, 14, 12, 11, 9, 8, 8, 6, 5, 3, 2, 2, 2, 3, 3, 5, 5, 6, 6, 7, 8, 9, 12, 15, 16, 16, 16, + 16, 17, 18, 17, 16, 15, 14, 13, 11, 10, 9, 9, 8, 6, 5, 3, 3, 3, 3, 4, 6, 6, 6, 6, 7, 8, 11, 14, 16, 16, 16, 16, + 17, 18, 18, 18, 17, 16, 16, 14, 12, 11, 10, 9, 8, 7, 6, 5, 3, 3, 5, 6, 6, 6, 6, 6, 8, 10, 12, 16, 17, 17, 17, 16, + 18, 18, 18, 18, 18, 17, 16, 14, 13, 12, 11, 11, 8, 8, 6, 6, 5, 4, 5, 5, 6, 6, 6, 7, 8, 11, 15, 16, 17, 17, 16, 16, + 18, 19, 19, 19, 19, 17, 17, 15, 14, 13, 12, 11, 11, 8, 7, 6, 6, 5, 5, 5, 7, 7, 7, 8, 9, 11, 15, 17, 17, 17, 16, 16, + 20, 20, 20, 20, 19, 19, 17, 17, 15, 14, 14, 12, 11, 9, 8, 7, 7, 6, 6, 6, 8, 8, 8, 8, 9, 12, 15, 18, 18, 16, 16, 16, + 22, 22, 22, 22, 21, 20, 19, 18, 17, 16, 15, 14, 12, 11, 10, 8, 8, 7, 7, 7, 8, 8, 8, 9, 10, 13, 17, 18, 18, 16, 16, 15, + 23, 22, 22, 22, 22, 21, 20, 20, 18, 18, 19, 16, 14, 13, 11, 10, 9, 8, 7, 7, 8, 9, 9, 10, 11, 15, 17, 18, 17, 17, 16, 14, + 23, 23, 23, 23, 23, 22, 21, 21, 20, 20, 20, 19, 18, 15, 12, 11, 10, 8, 8, 8, 9, 9, 10, 11, 13, 17, 17, 17, 17, 16, 15, 13, + 23, 23, 24, 24, 23, 23, 22, 21, 21, 21, 20, 20, 19, 17, 14, 12, 11, 9, 8, 9, 9, 10, 10, 12, 15, 17, 17, 17, 16, 16, 15, 13, + ] + - ct: 8500 + r: [ + 18, 17, 16, 15, 13, 12, 10, 9, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 12, 12, 8, 8, 10, 13, 14, 15, 17, 18, + 17, 17, 16, 14, 12, 11, 10, 10, 8, 8, 8, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 7, 7, 10, 13, 14, 15, 17, 17, + 17, 16, 15, 13, 12, 11, 10, 10, 8, 8, 7, 7, 7, 6, 7, 7, 8, 8, 8, 7, 7, 7, 7, 7, 6, 6, 9, 12, 14, 15, 17, 17, + 16, 16, 15, 13, 12, 11, 10, 10, 9, 7, 7, 7, 6, 6, 5, 5, 6, 6, 7, 7, 7, 7, 7, 5, 5, 5, 8, 11, 14, 15, 17, 19, + 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 7, 6, 6, 5, 5, 5, 5, 5, 5, 6, 6, 6, 5, 5, 5, 5, 7, 10, 13, 15, 17, 19, + 15, 15, 14, 13, 12, 11, 9, 9, 8, 7, 6, 5, 5, 5, 4, 4, 5, 5, 5, 4, 4, 4, 4, 4, 4, 5, 6, 9, 13, 15, 16, 17, + 15, 15, 13, 12, 11, 11, 9, 8, 8, 6, 5, 5, 4, 4, 4, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 4, 5, 8, 13, 15, 15, 16, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 4, 5, 7, 11, 14, 15, 15, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 4, 4, 4, 3, 3, 4, 3, 3, 3, 2, 3, 3, 3, 3, 4, 6, 10, 14, 15, 15, + 15, 13, 13, 11, 11, 10, 9, 8, 7, 6, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 3, 4, 6, 10, 13, 16, 16, + 14, 13, 12, 11, 10, 10, 9, 7, 7, 5, 4, 3, 3, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 3, 4, 6, 9, 13, 16, 17, + 14, 13, 12, 11, 10, 10, 9, 7, 6, 5, 4, 3, 3, 3, 2, 2, 2, 3, 3, 3, 2, 2, 1, 2, 2, 3, 4, 6, 9, 13, 16, 17, + 14, 13, 12, 12, 10, 10, 9, 7, 5, 5, 4, 3, 3, 2, 2, 2, 2, 3, 3, 3, 2, 2, 1, 1, 2, 3, 4, 6, 9, 13, 15, 17, + 14, 13, 12, 11, 10, 9, 9, 7, 5, 5, 4, 3, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 1, 1, 2, 3, 4, 6, 9, 12, 15, 16, + 13, 13, 12, 11, 10, 9, 8, 7, 5, 4, 4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 3, 4, 6, 8, 12, 15, 16, + 13, 13, 12, 11, 11, 9, 8, 7, 6, 4, 3, 2, 1, 1, 1, 1, 2, 2, 2, 2, 2, 1, 1, 2, 3, 3, 4, 6, 9, 15, 16, 16, + 13, 13, 12, 12, 11, 10, 9, 7, 6, 4, 3, 2, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 4, 4, 6, 10, 15, 17, 18, + 14, 13, 13, 12, 12, 11, 9, 7, 6, 4, 3, 2, 1, 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 4, 5, 7, 11, 15, 17, 18, + 14, 13, 13, 12, 12, 11, 9, 7, 7, 4, 3, 1, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 4, 4, 5, 7, 11, 15, 17, 18, + 15, 13, 13, 12, 12, 11, 8, 7, 6, 4, 3, 1, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 4, 5, 6, 7, 11, 15, 17, 18, + 15, 14, 13, 13, 12, 11, 8, 8, 7, 5, 4, 2, 1, 0, 0, 0, 0, 1, 2, 2, 2, 2, 3, 3, 5, 5, 7, 8, 11, 15, 17, 18, + 15, 14, 14, 13, 12, 11, 9, 8, 7, 6, 5, 3, 2, 1, 0, 0, 0, 1, 2, 2, 3, 3, 3, 5, 5, 6, 8, 9, 13, 16, 18, 18, + 15, 14, 14, 13, 12, 11, 9, 9, 8, 7, 6, 5, 3, 3, 1, 0, 0, 1, 2, 2, 3, 4, 4, 5, 5, 6, 8, 11, 14, 17, 18, 18, + 16, 16, 15, 14, 13, 12, 10, 9, 8, 8, 7, 6, 5, 5, 3, 1, 1, 1, 2, 3, 4, 4, 5, 5, 6, 8, 10, 13, 15, 17, 18, 18, + 16, 16, 15, 14, 14, 12, 11, 10, 9, 9, 8, 7, 6, 5, 4, 3, 1, 1, 2, 4, 4, 5, 5, 5, 7, 9, 11, 14, 16, 18, 18, 19, + 16, 16, 15, 15, 14, 13, 12, 11, 10, 10, 9, 8, 7, 6, 5, 4, 3, 2, 3, 3, 4, 5, 5, 6, 9, 10, 13, 15, 18, 18, 19, 19, + 17, 17, 16, 15, 15, 14, 12, 11, 11, 10, 10, 9, 8, 7, 6, 5, 4, 3, 3, 3, 5, 5, 6, 6, 9, 11, 13, 17, 18, 19, 19, 20, + 19, 18, 17, 17, 15, 15, 13, 12, 11, 11, 10, 10, 9, 8, 7, 5, 5, 4, 4, 5, 6, 6, 6, 7, 10, 12, 15, 18, 19, 19, 19, 20, + 19, 19, 18, 17, 17, 16, 14, 13, 12, 12, 11, 10, 9, 8, 7, 6, 6, 5, 5, 6, 6, 6, 7, 8, 11, 12, 16, 18, 19, 19, 19, 19, + 20, 19, 19, 18, 17, 16, 15, 14, 13, 13, 12, 12, 10, 9, 8, 7, 6, 5, 5, 6, 6, 7, 8, 9, 11, 14, 17, 18, 19, 19, 19, 19, + 20, 20, 19, 18, 18, 17, 15, 15, 14, 14, 14, 13, 12, 11, 9, 7, 6, 5, 5, 6, 7, 7, 8, 9, 12, 15, 17, 18, 18, 19, 18, 18, + 21, 20, 20, 19, 18, 18, 15, 15, 14, 14, 14, 14, 13, 13, 11, 9, 6, 5, 5, 6, 7, 7, 9, 10, 13, 15, 18, 18, 18, 18, 18, 18, + ] + g: [ + 16, 16, 15, 13, 12, 10, 9, 8, 7, 7, 7, 6, 6, 6, 7, 7, 7, 6, 6, 6, 6, 6, 11, 11, 6, 6, 9, 12, 12, 13, 15, 15, + 16, 15, 14, 13, 11, 10, 9, 9, 8, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 9, 12, 12, 13, 15, 15, + 15, 15, 14, 12, 11, 11, 9, 9, 8, 7, 6, 6, 6, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 5, 4, 5, 8, 11, 13, 14, 15, 15, + 15, 15, 14, 12, 11, 10, 9, 9, 8, 6, 6, 6, 5, 5, 4, 4, 4, 4, 6, 6, 6, 6, 6, 4, 4, 4, 7, 11, 13, 14, 15, 17, + 15, 15, 13, 12, 11, 10, 9, 9, 8, 6, 6, 5, 5, 4, 4, 3, 3, 4, 4, 5, 5, 5, 4, 4, 3, 4, 6, 9, 13, 14, 15, 17, + 15, 14, 13, 12, 11, 10, 9, 8, 8, 6, 5, 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 3, 4, 5, 8, 13, 14, 14, 15, + 14, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 4, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 4, 7, 12, 14, 14, 14, + 14, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 3, 3, 4, 6, 11, 14, 14, 14, + 14, 13, 12, 11, 11, 10, 9, 8, 7, 6, 4, 4, 3, 3, 3, 2, 3, 3, 3, 3, 2, 2, 2, 2, 2, 3, 3, 5, 10, 14, 14, 14, + 15, 13, 12, 11, 11, 10, 9, 8, 7, 6, 4, 4, 3, 3, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 5, 10, 13, 15, 16, + 14, 13, 12, 11, 11, 10, 9, 7, 7, 5, 4, 3, 2, 2, 2, 2, 2, 2, 3, 2, 2, 1, 2, 2, 2, 2, 3, 6, 9, 13, 16, 16, + 14, 13, 12, 11, 10, 10, 9, 7, 5, 5, 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 4, 6, 8, 13, 16, 16, + 14, 13, 13, 12, 10, 10, 9, 7, 5, 5, 4, 3, 2, 2, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 2, 2, 4, 5, 8, 13, 15, 16, + 14, 13, 13, 12, 10, 10, 9, 7, 5, 5, 4, 3, 2, 1, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 3, 5, 8, 12, 14, 15, + 14, 13, 12, 11, 10, 10, 9, 7, 6, 4, 3, 2, 1, 1, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 2, 3, 3, 5, 8, 12, 15, 15, + 14, 13, 12, 11, 11, 10, 9, 7, 6, 4, 3, 2, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 2, 2, 3, 4, 5, 8, 14, 16, 16, + 14, 13, 13, 12, 12, 10, 9, 7, 6, 4, 3, 2, 1, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 5, 10, 15, 16, 17, + 14, 13, 13, 13, 12, 12, 9, 7, 7, 4, 3, 2, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 4, 6, 11, 15, 17, 17, + 14, 14, 13, 12, 12, 12, 9, 8, 7, 5, 3, 2, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5, 7, 11, 15, 17, 17, + 15, 14, 13, 13, 12, 12, 9, 8, 7, 5, 3, 2, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5, 5, 7, 11, 15, 17, 17, + 16, 14, 14, 13, 13, 12, 9, 8, 7, 5, 4, 2, 1, 0, 0, 0, 0, 1, 2, 2, 3, 3, 3, 3, 5, 5, 6, 8, 12, 16, 17, 17, + 16, 15, 14, 14, 13, 12, 10, 9, 7, 6, 5, 4, 2, 1, 0, 0, 0, 1, 2, 3, 3, 3, 3, 5, 5, 6, 8, 9, 13, 17, 17, 17, + 16, 15, 15, 14, 13, 12, 10, 10, 8, 7, 6, 5, 4, 3, 2, 0, 1, 1, 2, 3, 3, 4, 5, 5, 5, 6, 8, 11, 14, 17, 17, 17, + 16, 16, 15, 15, 14, 13, 11, 10, 9, 8, 7, 6, 5, 5, 3, 2, 1, 1, 2, 3, 5, 5, 5, 5, 6, 7, 10, 13, 16, 17, 17, 17, + 17, 17, 15, 15, 14, 13, 12, 11, 10, 9, 9, 7, 6, 6, 5, 3, 2, 2, 3, 4, 5, 5, 5, 6, 6, 9, 11, 14, 16, 18, 18, 18, + 17, 17, 16, 15, 15, 14, 13, 12, 11, 10, 10, 9, 7, 7, 5, 4, 3, 2, 3, 4, 5, 5, 5, 6, 9, 10, 13, 15, 18, 18, 18, 18, + 18, 17, 17, 16, 15, 15, 14, 12, 11, 11, 11, 9, 8, 7, 6, 5, 4, 3, 3, 4, 5, 5, 6, 6, 9, 11, 13, 17, 18, 18, 18, 18, + 20, 19, 18, 18, 16, 16, 14, 13, 12, 11, 11, 10, 9, 8, 7, 5, 5, 4, 4, 5, 6, 6, 6, 7, 10, 11, 15, 18, 18, 18, 18, 18, + 20, 20, 19, 18, 18, 17, 15, 14, 13, 13, 12, 11, 10, 9, 8, 6, 6, 5, 5, 6, 6, 7, 7, 8, 11, 12, 16, 18, 18, 18, 18, 18, + 22, 21, 20, 19, 18, 17, 17, 17, 15, 15, 14, 13, 11, 10, 8, 7, 6, 6, 6, 6, 7, 7, 8, 8, 11, 14, 17, 18, 18, 18, 18, 17, + 22, 22, 21, 20, 19, 18, 17, 17, 17, 16, 16, 15, 14, 12, 10, 8, 7, 6, 6, 7, 7, 8, 8, 10, 13, 16, 18, 18, 18, 18, 18, 16, + 22, 22, 22, 21, 20, 19, 18, 17, 17, 17, 16, 16, 15, 15, 12, 10, 7, 6, 6, 7, 8, 8, 9, 11, 14, 16, 18, 18, 18, 18, 17, 16, + ] + b: [ + 13, 13, 13, 11, 10, 9, 9, 8, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 9, 9, 4, 4, 7, 9, 9, 9, 10, 10, + 13, 13, 12, 11, 10, 9, 9, 8, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 7, 9, 9, 10, 10, 10, + 13, 13, 12, 11, 10, 10, 9, 9, 7, 7, 6, 6, 6, 5, 5, 5, 5, 6, 6, 6, 5, 5, 5, 4, 3, 3, 6, 9, 10, 11, 11, 11, + 13, 13, 12, 11, 10, 10, 9, 9, 8, 7, 6, 6, 5, 5, 4, 4, 4, 4, 5, 5, 5, 5, 5, 3, 2, 3, 5, 9, 10, 11, 11, 13, + 13, 13, 12, 11, 11, 10, 9, 9, 8, 7, 6, 5, 5, 4, 4, 3, 3, 4, 4, 5, 5, 5, 3, 2, 2, 3, 5, 8, 11, 11, 12, 13, + 13, 12, 12, 11, 11, 10, 9, 8, 8, 7, 5, 5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 3, 7, 11, 11, 11, 11, + 13, 13, 12, 11, 11, 10, 9, 8, 8, 7, 5, 4, 4, 3, 3, 3, 3, 3, 3, 2, 2, 3, 2, 2, 2, 2, 3, 6, 11, 11, 11, 11, + 13, 13, 12, 11, 11, 11, 9, 8, 7, 6, 5, 4, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 5, 10, 12, 12, 11, + 13, 13, 13, 11, 11, 11, 10, 8, 7, 6, 5, 4, 3, 3, 3, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 4, 9, 12, 12, 12, + 14, 13, 13, 12, 11, 11, 10, 8, 7, 6, 4, 4, 3, 2, 2, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 4, 9, 12, 13, 14, + 13, 13, 12, 12, 11, 11, 10, 8, 7, 6, 4, 3, 2, 2, 2, 1, 2, 3, 3, 2, 2, 1, 2, 2, 2, 2, 2, 4, 8, 12, 14, 14, + 13, 13, 12, 12, 11, 11, 11, 8, 7, 6, 5, 3, 2, 2, 1, 1, 2, 3, 3, 2, 2, 2, 2, 2, 2, 2, 3, 4, 8, 12, 14, 14, + 13, 13, 12, 12, 12, 11, 11, 8, 7, 6, 5, 3, 2, 1, 1, 1, 1, 2, 3, 3, 2, 2, 2, 2, 2, 3, 3, 4, 8, 12, 14, 14, + 13, 13, 13, 12, 12, 11, 11, 8, 6, 6, 5, 3, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 4, 7, 12, 13, 14, + 13, 13, 13, 12, 12, 11, 10, 8, 6, 5, 4, 2, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 5, 7, 12, 14, 14, + 14, 14, 13, 13, 13, 11, 10, 8, 7, 5, 4, 2, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 4, 4, 5, 8, 14, 16, 16, + 14, 14, 13, 13, 13, 12, 11, 9, 7, 5, 4, 2, 1, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 6, 10, 15, 16, 16, + 15, 14, 14, 14, 15, 15, 11, 9, 8, 5, 4, 2, 1, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 7, 11, 15, 16, 17, + 15, 15, 14, 14, 15, 15, 12, 10, 9, 6, 4, 2, 1, 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 5, 5, 6, 7, 11, 15, 16, 16, + 15, 15, 15, 15, 15, 15, 12, 10, 9, 6, 5, 2, 1, 0, 0, 0, 1, 1, 1, 3, 3, 3, 4, 4, 5, 6, 6, 7, 11, 15, 16, 17, + 16, 16, 15, 16, 15, 15, 12, 11, 10, 7, 5, 3, 2, 1, 0, 0, 1, 1, 2, 3, 4, 4, 4, 5, 6, 6, 7, 9, 12, 16, 16, 16, + 16, 16, 16, 16, 16, 15, 13, 12, 10, 9, 6, 5, 3, 2, 1, 1, 1, 2, 3, 4, 4, 4, 5, 6, 6, 7, 8, 9, 14, 16, 17, 16, + 16, 16, 16, 16, 16, 15, 14, 12, 11, 10, 8, 6, 5, 4, 2, 1, 1, 2, 3, 4, 5, 5, 6, 6, 6, 7, 9, 12, 15, 16, 16, 16, + 17, 17, 18, 17, 16, 16, 14, 13, 12, 11, 9, 8, 6, 5, 4, 2, 2, 2, 3, 4, 6, 6, 6, 6, 7, 8, 11, 14, 16, 17, 16, 16, + 17, 17, 18, 18, 17, 16, 16, 14, 13, 12, 11, 9, 8, 7, 5, 4, 2, 3, 4, 6, 6, 6, 6, 7, 8, 10, 12, 15, 17, 17, 17, 16, + 18, 18, 18, 18, 18, 17, 16, 15, 14, 13, 12, 11, 9, 8, 6, 5, 4, 4, 4, 5, 6, 6, 7, 7, 10, 11, 14, 16, 17, 17, 17, 17, + 18, 18, 19, 19, 19, 18, 17, 16, 15, 14, 14, 11, 10, 9, 7, 6, 6, 5, 5, 5, 6, 7, 7, 7, 10, 12, 14, 17, 17, 17, 17, 17, + 20, 20, 20, 20, 20, 19, 18, 17, 16, 15, 14, 13, 11, 10, 9, 7, 6, 6, 6, 6, 7, 7, 7, 8, 11, 12, 15, 18, 18, 17, 17, 16, + 22, 21, 21, 21, 21, 21, 20, 19, 17, 16, 16, 14, 13, 11, 10, 8, 7, 7, 7, 7, 8, 8, 8, 9, 11, 13, 17, 18, 18, 17, 16, 15, + 23, 22, 22, 22, 22, 21, 21, 20, 19, 19, 18, 16, 14, 13, 11, 9, 8, 8, 8, 8, 8, 8, 9, 10, 12, 15, 18, 18, 18, 17, 16, 14, + 23, 24, 24, 23, 23, 22, 22, 21, 21, 20, 20, 19, 18, 15, 13, 11, 9, 8, 8, 8, 9, 9, 10, 11, 13, 16, 18, 18, 17, 17, 15, 14, + 24, 24, 24, 24, 24, 23, 22, 22, 22, 22, 20, 20, 19, 18, 15, 13, 10, 9, 9, 9, 9, 10, 11, 12, 15, 17, 18, 18, 17, 16, 15, 13, + ] +... diff --git a/src/ipa/mali-c55/data/meson.build b/src/ipa/mali-c55/data/meson.build new file mode 100644 index 00000000..8a5fdd36 --- /dev/null +++ b/src/ipa/mali-c55/data/meson.build @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: CC0-1.0 + +conf_files = files([ + 'imx415.yaml', + 'uncalibrated.yaml' +]) + +install_data(conf_files, + install_dir : ipa_data_dir / 'mali-c55') diff --git a/src/ipa/mali-c55/data/uncalibrated.yaml b/src/ipa/mali-c55/data/uncalibrated.yaml new file mode 100644 index 00000000..6dcc0295 --- /dev/null +++ b/src/ipa/mali-c55/data/uncalibrated.yaml @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: CC0-1.0 +%YAML 1.1 +--- +version: 1 +algorithms: + - Agc: +... diff --git a/src/ipa/mali-c55/ipa_context.cpp b/src/ipa/mali-c55/ipa_context.cpp new file mode 100644 index 00000000..99f76ecd --- /dev/null +++ b/src/ipa/mali-c55/ipa_context.cpp @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * ipa_context.cpp - MaliC55 IPA Context + */ + +#include "ipa_context.h" + +/** + * \file ipa_context.h + * \brief Context and state information shared between the algorithms + */ + +namespace libcamera::ipa::mali_c55 { + +/** + * \struct IPASessionConfiguration + * \brief Session configuration for the IPA module + * + * The session configuration contains all IPA configuration parameters that + * remain constant during the capture session, from IPA module start to stop. + * It is typically set during the configure() operation of the IPA module, but + * may also be updated in the start() operation. + */ + +/** + * \struct IPAActiveState + * \brief Active state for algorithms + * + * The active state contains all algorithm-specific data that needs to be + * maintained by algorithms across frames. Unlike the session configuration, + * the active state is mutable and constantly updated by algorithms. The active + * state is accessible through the IPAContext structure. + * + * The active state stores two distinct categories of information: + * + * - The consolidated value of all algorithm controls. Requests passed to + * the queueRequest() function store values for controls that the + * application wants to modify for that particular frame, and the + * queueRequest() function updates the active state with those values. + * The active state thus contains a consolidated view of the value of all + * controls handled by the algorithm. + * + * - The value of parameters computed by the algorithm when running in auto + * mode. Algorithms running in auto mode compute new parameters every + * time statistics buffers are received (either synchronously, or + * possibly in a background thread). The latest computed value of those + * parameters is stored in the active state in the process() function. + * + * Each of the members in the active state belongs to a specific algorithm. A + * member may be read by any algorithm, but shall only be written by its owner. + */ + +/** + * \struct IPAFrameContext + * \brief Per-frame context for algorithms + * + * The frame context stores two distinct categories of information: + * + * - The value of the controls to be applied to the frame. These values are + * typically set in the queueRequest() function, from the consolidated + * control values stored in the active state. The frame context thus stores + * values for all controls related to the algorithm, not limited to the + * controls specified in the corresponding request, but consolidated from all + * requests that have been queued so far. + * + * For controls that can be set manually or computed by an algorithm + * (depending on the algorithm operation mode), such as for instance the + * colour gains for the AWB algorithm, the control value will be stored in + * the frame context in the queueRequest() function only when operating in + * manual mode. When operating in auto mode, the values are computed by the + * algorithm in process(), stored in the active state, and copied to the + * frame context in prepare(), just before being stored in the ISP parameters + * buffer. + * + * The queueRequest() function can also store ancillary data in the frame + * context, such as flags to indicate if (and what) control values have + * changed compared to the previous request. + * + * - Status information computed by the algorithm for a frame. For instance, + * the colour temperature estimated by the AWB algorithm from ISP statistics + * calculated on a frame is stored in the frame context for that frame in + * the process() function. + */ + +/** + * \struct IPAContext + * \brief Global IPA context data shared between all algorithms + * + * \var IPAContext::configuration + * \brief The IPA session configuration, immutable during the session + * + * \var IPAContext::activeState + * \brief The IPA active state, storing the latest state for all algorithms + * + * \var IPAContext::frameContexts + * \brief Ring buffer of per-frame contexts + */ + +} /* namespace libcamera::ipa::mali_c55 */ diff --git a/src/ipa/mali-c55/ipa_context.h b/src/ipa/mali-c55/ipa_context.h new file mode 100644 index 00000000..5e3e2fbd --- /dev/null +++ b/src/ipa/mali-c55/ipa_context.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * ipa_context.h - Mali-C55 IPA Context + */ + +#pragma once + +#include <libcamera/base/utils.h> +#include <libcamera/controls.h> + +#include "libcamera/internal/bayer_format.h" + +#include <libipa/fc_queue.h> + +namespace libcamera { + +namespace ipa::mali_c55 { + +struct IPASessionConfiguration { + struct { + utils::Duration minShutterSpeed; + utils::Duration maxShutterSpeed; + uint32_t defaultExposure; + double minAnalogueGain; + double maxAnalogueGain; + } agc; + + struct { + BayerFormat::Order bayerOrder; + utils::Duration lineDuration; + uint32_t blackLevel; + } sensor; +}; + +struct IPAActiveState { + struct { + struct { + uint32_t exposure; + double sensorGain; + double ispGain; + } automatic; + struct { + uint32_t exposure; + double sensorGain; + double ispGain; + } manual; + bool autoEnabled; + uint32_t constraintMode; + uint32_t exposureMode; + uint32_t temperatureK; + } agc; + + struct { + double rGain; + double bGain; + } awb; +}; + +struct IPAFrameContext : public FrameContext { + struct { + uint32_t exposure; + double sensorGain; + double ispGain; + } agc; + + struct { + double rGain; + double bGain; + } awb; +}; + +struct IPAContext { + IPAContext(unsigned int frameContextSize) + : frameContexts(frameContextSize) + { + } + + IPASessionConfiguration configuration; + IPAActiveState activeState; + + FCQueue<IPAFrameContext> frameContexts; + + ControlInfoMap::Map ctrlMap; +}; + +} /* namespace ipa::mali_c55 */ + +} /* namespace libcamera*/ diff --git a/src/ipa/mali-c55/mali-c55.cpp b/src/ipa/mali-c55/mali-c55.cpp new file mode 100644 index 00000000..c6941a95 --- /dev/null +++ b/src/ipa/mali-c55/mali-c55.cpp @@ -0,0 +1,399 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2023, Ideas on Board Oy + * + * mali-c55.cpp - Mali-C55 ISP image processing algorithms + */ + +#include <map> +#include <string.h> +#include <vector> + +#include <linux/mali-c55-config.h> +#include <linux/v4l2-controls.h> + +#include <libcamera/base/file.h> +#include <libcamera/base/log.h> + +#include <libcamera/control_ids.h> +#include <libcamera/ipa/ipa_interface.h> +#include <libcamera/ipa/ipa_module_info.h> +#include <libcamera/ipa/mali-c55_ipa_interface.h> + +#include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/mapped_framebuffer.h" +#include "libcamera/internal/yaml_parser.h" + +#include "algorithms/algorithm.h" +#include "libipa/camera_sensor_helper.h" + +#include "ipa_context.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPAMaliC55) + +using namespace std::literals::chrono_literals; + +namespace ipa::mali_c55 { + +/* Maximum number of frame contexts to be held */ +static constexpr uint32_t kMaxFrameContexts = 16; + +class IPAMaliC55 : public IPAMaliC55Interface, public Module +{ +public: + IPAMaliC55(); + + int init(const IPASettings &settings, const IPAConfigInfo &ipaConfig, + ControlInfoMap *ipaControls) override; + int start() override; + void stop() override; + int configure(const IPAConfigInfo &ipaConfig, uint8_t bayerOrder, + ControlInfoMap *ipaControls) override; + void mapBuffers(const std::vector<IPABuffer> &buffers, bool readOnly) override; + void unmapBuffers(const std::vector<IPABuffer> &buffers) override; + void queueRequest(const uint32_t request, const ControlList &controls) override; + void fillParams(unsigned int request, uint32_t bufferId) override; + void processStats(unsigned int request, unsigned int bufferId, + const ControlList &sensorControls) override; + +protected: + std::string logPrefix() const override; + +private: + void updateSessionConfiguration(const IPACameraSensorInfo &info, + const ControlInfoMap &sensorControls, + BayerFormat::Order bayerOrder); + void updateControls(const IPACameraSensorInfo &sensorInfo, + const ControlInfoMap &sensorControls, + ControlInfoMap *ipaControls); + void setControls(); + + std::map<unsigned int, MappedFrameBuffer> buffers_; + + ControlInfoMap sensorControls_; + + /* Interface to the Camera Helper */ + std::unique_ptr<CameraSensorHelper> camHelper_; + + /* Local parameter storage */ + struct IPAContext context_; +}; + +namespace { + +} /* namespace */ + +IPAMaliC55::IPAMaliC55() + : context_(kMaxFrameContexts) +{ +} + +std::string IPAMaliC55::logPrefix() const +{ + return "mali-c55"; +} + +int IPAMaliC55::init(const IPASettings &settings, const IPAConfigInfo &ipaConfig, + ControlInfoMap *ipaControls) +{ + camHelper_ = CameraSensorHelperFactoryBase::create(settings.sensorModel); + if (!camHelper_) { + LOG(IPAMaliC55, Error) + << "Failed to create camera sensor helper for " + << settings.sensorModel; + return -ENODEV; + } + + File file(settings.configurationFile); + if (!file.open(File::OpenModeFlag::ReadOnly)) { + int ret = file.error(); + LOG(IPAMaliC55, Error) + << "Failed to open configuration file " + << settings.configurationFile << ": " << strerror(-ret); + return ret; + } + + std::unique_ptr<libcamera::YamlObject> data = YamlParser::parse(file); + if (!data) + return -EINVAL; + + if (!data->contains("algorithms")) { + LOG(IPAMaliC55, Error) + << "Tuning file doesn't contain any algorithm"; + return -EINVAL; + } + + int ret = createAlgorithms(context_, (*data)["algorithms"]); + if (ret) + return ret; + + updateControls(ipaConfig.sensorInfo, ipaConfig.sensorControls, ipaControls); + + return 0; +} + +void IPAMaliC55::setControls() +{ + IPAActiveState &activeState = context_.activeState; + uint32_t exposure; + uint32_t gain; + + if (activeState.agc.autoEnabled) { + exposure = activeState.agc.automatic.exposure; + gain = camHelper_->gainCode(activeState.agc.automatic.sensorGain); + } else { + exposure = activeState.agc.manual.exposure; + gain = camHelper_->gainCode(activeState.agc.manual.sensorGain); + } + + ControlList ctrls(sensorControls_); + ctrls.set(V4L2_CID_EXPOSURE, static_cast<int32_t>(exposure)); + ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast<int32_t>(gain)); + + setSensorControls.emit(ctrls); +} + +int IPAMaliC55::start() +{ + return 0; +} + +void IPAMaliC55::stop() +{ + context_.frameContexts.clear(); +} + +void IPAMaliC55::updateSessionConfiguration(const IPACameraSensorInfo &info, + const ControlInfoMap &sensorControls, + BayerFormat::Order bayerOrder) +{ + context_.configuration.sensor.bayerOrder = bayerOrder; + + const ControlInfo &v4l2Exposure = sensorControls.find(V4L2_CID_EXPOSURE)->second; + int32_t minExposure = v4l2Exposure.min().get<int32_t>(); + int32_t maxExposure = v4l2Exposure.max().get<int32_t>(); + int32_t defExposure = v4l2Exposure.def().get<int32_t>(); + + const ControlInfo &v4l2Gain = sensorControls.find(V4L2_CID_ANALOGUE_GAIN)->second; + int32_t minGain = v4l2Gain.min().get<int32_t>(); + int32_t maxGain = v4l2Gain.max().get<int32_t>(); + + /* + * When the AGC computes the new exposure values for a frame, it needs + * to know the limits for shutter speed and analogue gain. + * As it depends on the sensor, update it with the controls. + * + * \todo take VBLANK into account for maximum shutter speed + */ + context_.configuration.sensor.lineDuration = info.minLineLength * 1.0s / info.pixelRate; + context_.configuration.agc.minShutterSpeed = minExposure * context_.configuration.sensor.lineDuration; + context_.configuration.agc.maxShutterSpeed = maxExposure * context_.configuration.sensor.lineDuration; + context_.configuration.agc.defaultExposure = defExposure; + context_.configuration.agc.minAnalogueGain = camHelper_->gain(minGain); + context_.configuration.agc.maxAnalogueGain = camHelper_->gain(maxGain); + + if (camHelper_->blackLevel().has_value()) { + /* + * The black level from CameraSensorHelper is a 16-bit value. + * The Mali-C55 ISP expects 20-bit settings, so we shift it to + * the appropriate width + */ + context_.configuration.sensor.blackLevel = + camHelper_->blackLevel().value() << 4; + } +} + +void IPAMaliC55::updateControls(const IPACameraSensorInfo &sensorInfo, + const ControlInfoMap &sensorControls, + ControlInfoMap *ipaControls) +{ + ControlInfoMap::Map ctrlMap; + + /* + * Compute the frame duration limits. + * + * The frame length is computed assuming a fixed line length combined + * with the vertical frame sizes. + */ + const ControlInfo &v4l2HBlank = sensorControls.find(V4L2_CID_HBLANK)->second; + uint32_t hblank = v4l2HBlank.def().get<int32_t>(); + uint32_t lineLength = sensorInfo.outputSize.width + hblank; + + const ControlInfo &v4l2VBlank = sensorControls.find(V4L2_CID_VBLANK)->second; + std::array<uint32_t, 3> frameHeights{ + v4l2VBlank.min().get<int32_t>() + sensorInfo.outputSize.height, + v4l2VBlank.max().get<int32_t>() + sensorInfo.outputSize.height, + v4l2VBlank.def().get<int32_t>() + sensorInfo.outputSize.height, + }; + + std::array<int64_t, 3> frameDurations; + for (unsigned int i = 0; i < frameHeights.size(); ++i) { + uint64_t frameSize = lineLength * frameHeights[i]; + frameDurations[i] = frameSize / (sensorInfo.pixelRate / 1000000U); + } + + ctrlMap[&controls::FrameDurationLimits] = ControlInfo(frameDurations[0], + frameDurations[1], + frameDurations[2]); + + /* + * Compute exposure time limits from the V4L2_CID_EXPOSURE control + * limits and the line duration. + */ + double lineDuration = sensorInfo.minLineLength / sensorInfo.pixelRate; + + const ControlInfo &v4l2Exposure = sensorControls.find(V4L2_CID_EXPOSURE)->second; + int32_t minExposure = v4l2Exposure.min().get<int32_t>() * lineDuration; + int32_t maxExposure = v4l2Exposure.max().get<int32_t>() * lineDuration; + int32_t defExposure = v4l2Exposure.def().get<int32_t>() * lineDuration; + ctrlMap[&controls::ExposureTime] = ControlInfo(minExposure, maxExposure, defExposure); + + /* Compute the analogue gain limits. */ + const ControlInfo &v4l2Gain = sensorControls.find(V4L2_CID_ANALOGUE_GAIN)->second; + float minGain = camHelper_->gain(v4l2Gain.min().get<int32_t>()); + float maxGain = camHelper_->gain(v4l2Gain.max().get<int32_t>()); + float defGain = camHelper_->gain(v4l2Gain.def().get<int32_t>()); + ctrlMap[&controls::AnalogueGain] = ControlInfo(minGain, maxGain, defGain); + + /* + * Merge in any controls that we support either statically or from the + * algorithms. + */ + ctrlMap.merge(context_.ctrlMap); + + *ipaControls = ControlInfoMap(std::move(ctrlMap), controls::controls); +} + +int IPAMaliC55::configure(const IPAConfigInfo &ipaConfig, uint8_t bayerOrder, + ControlInfoMap *ipaControls) +{ + sensorControls_ = ipaConfig.sensorControls; + + /* Clear the IPA context before the streaming session. */ + context_.configuration = {}; + context_.activeState = {}; + context_.frameContexts.clear(); + + const IPACameraSensorInfo &info = ipaConfig.sensorInfo; + + updateSessionConfiguration(info, ipaConfig.sensorControls, + static_cast<BayerFormat::Order>(bayerOrder)); + updateControls(info, ipaConfig.sensorControls, ipaControls); + + for (auto const &a : algorithms()) { + Algorithm *algo = static_cast<Algorithm *>(a.get()); + + int ret = algo->configure(context_, info); + if (ret) + return ret; + } + + return 0; +} + +void IPAMaliC55::mapBuffers(const std::vector<IPABuffer> &buffers, bool readOnly) +{ + for (const IPABuffer &buffer : buffers) { + const FrameBuffer fb(buffer.planes); + buffers_.emplace( + buffer.id, + MappedFrameBuffer( + &fb, + readOnly ? MappedFrameBuffer::MapFlag::Read + : MappedFrameBuffer::MapFlag::ReadWrite)); + } +} + +void IPAMaliC55::unmapBuffers(const std::vector<IPABuffer> &buffers) +{ + for (const IPABuffer &buffer : buffers) { + auto it = buffers_.find(buffer.id); + if (it == buffers_.end()) + continue; + + buffers_.erase(buffer.id); + } +} + +void IPAMaliC55::queueRequest(const uint32_t request, const ControlList &controls) +{ + IPAFrameContext &frameContext = context_.frameContexts.alloc(request); + + for (auto const &a : algorithms()) { + Algorithm *algo = static_cast<Algorithm *>(a.get()); + + algo->queueRequest(context_, request, frameContext, controls); + } +} + +void IPAMaliC55::fillParams(unsigned int request, + [[maybe_unused]] uint32_t bufferId) +{ + struct mali_c55_params_buffer *params; + IPAFrameContext &frameContext = context_.frameContexts.get(request); + + params = reinterpret_cast<mali_c55_params_buffer *>( + buffers_.at(bufferId).planes()[0].data()); + memset(params, 0, sizeof(mali_c55_params_buffer)); + + params->version = MALI_C55_PARAM_BUFFER_V1; + + for (auto const &algo : algorithms()) { + algo->prepare(context_, request, frameContext, params); + + ASSERT(params->total_size <= MALI_C55_PARAMS_MAX_SIZE); + } + + paramsComputed.emit(request); +} + +void IPAMaliC55::processStats(unsigned int request, unsigned int bufferId, + const ControlList &sensorControls) +{ + IPAFrameContext &frameContext = context_.frameContexts.get(request); + const mali_c55_stats_buffer *stats = nullptr; + + stats = reinterpret_cast<mali_c55_stats_buffer *>( + buffers_.at(bufferId).planes()[0].data()); + + frameContext.agc.exposure = + sensorControls.get(V4L2_CID_EXPOSURE).get<int32_t>(); + frameContext.agc.sensorGain = + camHelper_->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get<int32_t>()); + + ControlList metadata(controls::controls); + + for (auto const &a : algorithms()) { + Algorithm *algo = static_cast<Algorithm *>(a.get()); + + algo->process(context_, request, frameContext, stats, metadata); + } + + setControls(); + + statsProcessed.emit(request, metadata); +} + +} /* namespace ipa::mali_c55 */ + +/* + * External IPA module interface + */ +extern "C" { +const struct IPAModuleInfo ipaModuleInfo = { + IPA_MODULE_API_VERSION, + 1, + "mali-c55", + "mali-c55", +}; + +IPAInterface *ipaCreate() +{ + return new ipa::mali_c55::IPAMaliC55(); +} + +} /* extern "C" */ + +} /* namespace libcamera */ diff --git a/src/ipa/mali-c55/meson.build b/src/ipa/mali-c55/meson.build new file mode 100644 index 00000000..864d90ec --- /dev/null +++ b/src/ipa/mali-c55/meson.build @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: CC0-1.0 + +subdir('algorithms') +subdir('data') + +ipa_name = 'ipa_mali_c55' + +mali_c55_ipa_sources = files([ + 'ipa_context.cpp', + 'mali-c55.cpp' +]) + +mali_c55_ipa_sources += mali_c55_ipa_algorithms + +mod = shared_module(ipa_name, + mali_c55_ipa_sources, + name_prefix : '', + include_directories : [ipa_includes, libipa_includes], + dependencies : libcamera_private, + link_with : libipa, + install : true, + install_dir : ipa_install_dir) + +if ipa_sign_module + custom_target(ipa_name + '.so.sign', + input : mod, + output : ipa_name + '.so.sign', + command : [ipa_sign, ipa_priv_key, '@INPUT@', '@OUTPUT@'], + install : false, + build_by_default : true) +endif + +ipa_names += ipa_name diff --git a/src/ipa/mali-c55/module.h b/src/ipa/mali-c55/module.h new file mode 100644 index 00000000..1d85ec1f --- /dev/null +++ b/src/ipa/mali-c55/module.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * module.h - Mali-C55 IPA Module + */ + +#pragma once + +#include <linux/mali-c55-config.h> + +#include <libcamera/ipa/mali-c55_ipa_interface.h> + +#include <libipa/module.h> + +#include "ipa_context.h" + +namespace libcamera { + +namespace ipa::mali_c55 { + +using Module = ipa::Module<IPAContext, IPAFrameContext, IPACameraSensorInfo, + mali_c55_params_buffer, mali_c55_stats_buffer>; + +} /* namespace ipa::mali_c55 */ + +} /* namespace libcamera*/ diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp index 40e5a8f4..9a558a1c 100644 --- a/src/ipa/rkisp1/algorithms/agc.cpp +++ b/src/ipa/rkisp1/algorithms/agc.cpp @@ -148,7 +148,16 @@ int Agc::init(IPAContext &context, const YamlObject &tuningData) if (ret) return ret; - context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true); + context.ctrlMap[&controls::ExposureTimeMode] = + ControlInfo({ { ControlValue(controls::ExposureTimeModeAuto), + ControlValue(controls::ExposureTimeModeManual) } }, + ControlValue(controls::ExposureTimeModeAuto)); + context.ctrlMap[&controls::AnalogueGainMode] = + ControlInfo({ { ControlValue(controls::AnalogueGainModeAuto), + ControlValue(controls::AnalogueGainModeManual) } }, + ControlValue(controls::AnalogueGainModeAuto)); + /* \todo Move this to the Camera class */ + context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap.merge(controls()); return 0; @@ -169,7 +178,8 @@ int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) 10ms / context.configuration.sensor.lineDuration; context.activeState.agc.manual.gain = context.activeState.agc.automatic.gain; context.activeState.agc.manual.exposure = context.activeState.agc.automatic.exposure; - context.activeState.agc.autoEnabled = !context.configuration.raw; + context.activeState.agc.autoExposureEnabled = !context.configuration.raw; + context.activeState.agc.autoGainEnabled = !context.configuration.raw; context.activeState.agc.constraintMode = static_cast<controls::AeConstraintModeEnum>(constraintModes().begin()->first); @@ -215,18 +225,47 @@ void Agc::queueRequest(IPAContext &context, auto &agc = context.activeState.agc; if (!context.configuration.raw) { - const auto &agcEnable = controls.get(controls::AeEnable); - if (agcEnable && *agcEnable != agc.autoEnabled) { - agc.autoEnabled = *agcEnable; + const auto &aeEnable = controls.get(controls::ExposureTimeMode); + if (aeEnable && + (*aeEnable == controls::ExposureTimeModeAuto) != agc.autoExposureEnabled) { + agc.autoExposureEnabled = (*aeEnable == controls::ExposureTimeModeAuto); LOG(RkISP1Agc, Debug) - << (agc.autoEnabled ? "Enabling" : "Disabling") - << " AGC"; + << (agc.autoExposureEnabled ? "Enabling" : "Disabling") + << " AGC (exposure)"; + + /* + * If we go from auto -> manual with no manual control + * set, use the last computed value, which we don't + * know until prepare() so save this information. + * + * \todo Check the previous frame at prepare() time + * instead of saving a flag here + */ + if (!agc.autoExposureEnabled && !controls.get(controls::ExposureTime)) + frameContext.agc.autoExposureModeChange = true; + } + + const auto &agEnable = controls.get(controls::AnalogueGainMode); + if (agEnable && + (*agEnable == controls::AnalogueGainModeAuto) != agc.autoGainEnabled) { + agc.autoGainEnabled = (*agEnable == controls::AnalogueGainModeAuto); + + LOG(RkISP1Agc, Debug) + << (agc.autoGainEnabled ? "Enabling" : "Disabling") + << " AGC (gain)"; + /* + * If we go from auto -> manual with no manual control + * set, use the last computed value, which we don't + * know until prepare() so save this information. + */ + if (!agc.autoGainEnabled && !controls.get(controls::AnalogueGain)) + frameContext.agc.autoGainModeChange = true; } } const auto &exposure = controls.get(controls::ExposureTime); - if (exposure && !agc.autoEnabled) { + if (exposure && !agc.autoExposureEnabled) { agc.manual.exposure = *exposure * 1.0us / context.configuration.sensor.lineDuration; @@ -235,18 +274,19 @@ void Agc::queueRequest(IPAContext &context, } const auto &gain = controls.get(controls::AnalogueGain); - if (gain && !agc.autoEnabled) { + if (gain && !agc.autoGainEnabled) { agc.manual.gain = *gain; LOG(RkISP1Agc, Debug) << "Set gain to " << agc.manual.gain; } - frameContext.agc.autoEnabled = agc.autoEnabled; + frameContext.agc.autoExposureEnabled = agc.autoExposureEnabled; + frameContext.agc.autoGainEnabled = agc.autoGainEnabled; - if (!frameContext.agc.autoEnabled) { + if (!frameContext.agc.autoExposureEnabled) frameContext.agc.exposure = agc.manual.exposure; + if (!frameContext.agc.autoGainEnabled) frameContext.agc.gain = agc.manual.gain; - } const auto &meteringMode = controls.get(controls::AeMeteringMode); if (meteringMode) { @@ -283,9 +323,26 @@ void Agc::queueRequest(IPAContext &context, void Agc::prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, RkISP1Params *params) { - if (frameContext.agc.autoEnabled) { - frameContext.agc.exposure = context.activeState.agc.automatic.exposure; - frameContext.agc.gain = context.activeState.agc.automatic.gain; + uint32_t activeAutoExposure = context.activeState.agc.automatic.exposure; + double activeAutoGain = context.activeState.agc.automatic.gain; + + /* Populate exposure and gain in auto mode */ + if (frameContext.agc.autoExposureEnabled) + frameContext.agc.exposure = activeAutoExposure; + if (frameContext.agc.autoGainEnabled) + frameContext.agc.gain = activeAutoGain; + + /* + * Populate manual exposure and gain from the active auto values when + * transitioning from auto to manual + */ + if (!frameContext.agc.autoExposureEnabled && frameContext.agc.autoExposureModeChange) { + context.activeState.agc.manual.exposure = activeAutoExposure; + frameContext.agc.exposure = activeAutoExposure; + } + if (!frameContext.agc.autoGainEnabled && frameContext.agc.autoGainModeChange) { + context.activeState.agc.manual.gain = activeAutoGain; + frameContext.agc.gain = activeAutoGain; } if (frame > 0 && !frameContext.agc.updateMetering) @@ -333,7 +390,14 @@ void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, * frameContext.sensor.exposure; metadata.set(controls::AnalogueGain, frameContext.sensor.gain); metadata.set(controls::ExposureTime, exposureTime.get<std::micro>()); - metadata.set(controls::AeEnable, frameContext.agc.autoEnabled); + metadata.set(controls::ExposureTimeMode, + frameContext.agc.autoExposureEnabled + ? controls::ExposureTimeModeAuto + : controls::ExposureTimeModeManual); + metadata.set(controls::AnalogueGainMode, + frameContext.agc.autoGainEnabled + ? controls::AnalogueGainModeAuto + : controls::AnalogueGainModeManual); /* \todo Use VBlank value calculated from each frame exposure. */ uint32_t vTotal = context.configuration.sensor.size.height @@ -402,7 +466,7 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, fillMetadata(context, frameContext, metadata); return; } - + if (!(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP)) { fillMetadata(context, frameContext, metadata); LOG(RkISP1Agc, Error) << "AUTOEXP data is missing in statistics"; @@ -424,14 +488,35 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, [](uint32_t x) { return x >> 4; }); expMeans_ = { params->ae.exp_mean, context.hw->numAeCells }; - utils::Duration maxExposureTime = - std::clamp(frameContext.agc.maxFrameDuration, - context.configuration.sensor.minExposureTime, - context.configuration.sensor.maxExposureTime); - setLimits(context.configuration.sensor.minExposureTime, - maxExposureTime, - context.configuration.sensor.minAnalogueGain, - context.configuration.sensor.maxAnalogueGain); + /* + * Set the AGC limits using the fixed exposure time and/or gain in + * manual mode, or the sensor limits in auto mode. + */ + utils::Duration minExposureTime; + utils::Duration maxExposureTime; + double minAnalogueGain; + double maxAnalogueGain; + + if (frameContext.agc.autoExposureEnabled) { + minExposureTime = context.configuration.sensor.minExposureTime; + maxExposureTime = std::clamp(frameContext.agc.maxFrameDuration, + context.configuration.sensor.minExposureTime, + context.configuration.sensor.maxExposureTime); + } else { + minExposureTime = context.configuration.sensor.lineDuration + * frameContext.agc.exposure; + maxExposureTime = minExposureTime; + } + + if (frameContext.agc.autoGainEnabled) { + minAnalogueGain = context.configuration.sensor.minAnalogueGain; + maxAnalogueGain = context.configuration.sensor.maxAnalogueGain; + } else { + minAnalogueGain = frameContext.agc.gain; + maxAnalogueGain = frameContext.agc.gain; + } + + setLimits(minExposureTime, maxExposureTime, minAnalogueGain, maxAnalogueGain); /* * The Agc algorithm needs to know the effective exposure value that was diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp index 4bb4f5b8..cffaa06a 100644 --- a/src/ipa/rkisp1/algorithms/awb.cpp +++ b/src/ipa/rkisp1/algorithms/awb.cpp @@ -33,6 +33,10 @@ namespace ipa::rkisp1::algorithms { LOG_DEFINE_CATEGORY(RkISP1Awb) +constexpr int32_t kMinColourTemperature = 2500; +constexpr int32_t kMaxColourTemperature = 10000; +constexpr int32_t kDefaultColourTemperature = 5000; + /* Minimum mean value below which AWB can't operate. */ constexpr double kMeanMinThreshold = 2.0; @@ -42,6 +46,29 @@ Awb::Awb() } /** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Awb::init(IPAContext &context, const YamlObject &tuningData) +{ + auto &cmap = context.ctrlMap; + cmap[&controls::ColourTemperature] = ControlInfo(kMinColourTemperature, + kMaxColourTemperature, + kDefaultColourTemperature); + + Interpolator<Vector<double, 2>> gainCurve; + int ret = gainCurve.readYaml(tuningData["colourGains"], "ct", "gains"); + if (ret < 0) + LOG(RkISP1Awb, Warning) + << "Failed to parse 'colourGains' " + << "parameter from tuning file; " + << "manual colour temperature will not work properly"; + else + colourGainCurve_ = gainCurve; + + return 0; +} + +/** * \copydoc libcamera::ipa::Algorithm::configure */ int Awb::configure(IPAContext &context, @@ -50,6 +77,7 @@ int Awb::configure(IPAContext &context, context.activeState.awb.gains.manual = RGB<double>{ 1.0 }; context.activeState.awb.gains.automatic = RGB<double>{ 1.0 }; context.activeState.awb.autoEnabled = true; + context.activeState.awb.temperatureK = kDefaultColourTemperature; /* * Define the measurement window for AWB as a centered rectangle @@ -83,19 +111,37 @@ void Awb::queueRequest(IPAContext &context, << (*awbEnable ? "Enabling" : "Disabling") << " AWB"; } + frameContext.awb.autoEnabled = awb.autoEnabled; + + if (awb.autoEnabled) + return; + const auto &colourGains = controls.get(controls::ColourGains); - if (colourGains && !awb.autoEnabled) { + const auto &colourTemperature = controls.get(controls::ColourTemperature); + bool update = false; + if (colourGains) { awb.gains.manual.r() = (*colourGains)[0]; awb.gains.manual.b() = (*colourGains)[1]; + /* + * \todo: Colour temperature reported in metadata is now + * incorrect, as we can't deduce the temperature from the gains. + * This will be fixed with the bayes AWB algorithm. + */ + update = true; + } else if (colourTemperature && colourGainCurve_) { + const auto &gains = colourGainCurve_->getInterpolated(*colourTemperature); + awb.gains.manual.r() = gains[0]; + awb.gains.manual.b() = gains[1]; + awb.temperatureK = *colourTemperature; + update = true; + } + if (update) LOG(RkISP1Awb, Debug) << "Set colour gains to " << awb.gains.manual; - } - - frameContext.awb.autoEnabled = awb.autoEnabled; - if (!awb.autoEnabled) - frameContext.awb.gains = awb.gains.manual; + frameContext.awb.gains = awb.gains.manual; + frameContext.awb.temperatureK = awb.temperatureK; } /** @@ -108,8 +154,10 @@ void Awb::prepare(IPAContext &context, const uint32_t frame, * This is the latest time we can read the active state. This is the * most up-to-date automatic values we can read. */ - if (frameContext.awb.autoEnabled) + if (frameContext.awb.autoEnabled) { frameContext.awb.gains = context.activeState.awb.gains.automatic; + frameContext.awb.temperatureK = context.activeState.awb.temperatureK; + } auto gainConfig = params->block<BlockType::AwbGain>(); gainConfig.setEnabled(true); @@ -188,7 +236,7 @@ void Awb::process(IPAContext &context, static_cast<float>(frameContext.awb.gains.r()), static_cast<float>(frameContext.awb.gains.b()) }); - metadata.set(controls::ColourTemperature, activeState.awb.temperatureK); + metadata.set(controls::ColourTemperature, frameContext.awb.temperatureK); if (!stats || !(stats->meas_type & RKISP1_CIF_ISP_STAT_AWB)) { LOG(RkISP1Awb, Error) << "AWB data is missing in statistics"; @@ -263,9 +311,6 @@ void Awb::process(IPAContext &context, activeState.awb.temperatureK = estimateCCT(rgbMeans); - /* Metadata shall contain the up to date measurement */ - metadata.set(controls::ColourTemperature, activeState.awb.temperatureK); - /* * Estimate the red and blue gains to apply in a grey world. The green * gain is hardcoded to 1.0. Avoid divisions by zero by clamping the diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h index 6ac3a5c3..34ec42cb 100644 --- a/src/ipa/rkisp1/algorithms/awb.h +++ b/src/ipa/rkisp1/algorithms/awb.h @@ -7,6 +7,12 @@ #pragma once +#include <optional> + +#include "libcamera/internal/vector.h" + +#include "libipa/interpolator.h" + #include "algorithm.h" namespace libcamera { @@ -19,6 +25,7 @@ public: Awb(); ~Awb() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; void queueRequest(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, @@ -32,6 +39,7 @@ public: ControlList &metadata) override; private: + std::optional<Interpolator<Vector<double, 2>>> colourGainCurve_; bool rgbMode_; }; diff --git a/src/ipa/rkisp1/algorithms/ccm.cpp b/src/ipa/rkisp1/algorithms/ccm.cpp index 6b7d2e2c..eb8ca39e 100644 --- a/src/ipa/rkisp1/algorithms/ccm.cpp +++ b/src/ipa/rkisp1/algorithms/ccm.cpp @@ -18,7 +18,7 @@ #include "libcamera/internal/yaml_parser.h" -#include "../utils.h" +#include "libipa/fixedpoint.h" #include "libipa/interpolator.h" /** @@ -72,7 +72,7 @@ void Ccm::setParameters(struct rkisp1_cif_isp_ctk_config &config, for (unsigned int i = 0; i < 3; i++) { for (unsigned int j = 0; j < 3; j++) config.coeff[i][j] = - utils::floatingToFixedPoint<4, 7, uint16_t, double>(matrix[i][j]); + floatingToFixedPoint<4, 7, uint16_t, double>(matrix[i][j]); } for (unsigned int i = 0; i < 3; i++) @@ -120,12 +120,7 @@ void Ccm::process([[maybe_unused]] IPAContext &context, [[maybe_unused]] const rkisp1_stat_buffer *stats, ControlList &metadata) { - float m[9]; - for (unsigned int i = 0; i < 3; i++) { - for (unsigned int j = 0; j < 3; j++) - m[i * 3 + j] = frameContext.ccm.ccm[i][j]; - } - metadata.set(controls::ColourCorrectionMatrix, m); + metadata.set(controls::ColourCorrectionMatrix, frameContext.ccm.ccm.data()); } REGISTER_IPA_ALGORITHM(Ccm, "Ccm") diff --git a/src/ipa/rkisp1/algorithms/lux.cpp b/src/ipa/rkisp1/algorithms/lux.cpp new file mode 100644 index 00000000..b0f74963 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/lux.cpp @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * lux.cpp - RkISP1 Lux control + */ + +#include "lux.h" + +#include <libcamera/base/log.h> + +#include <libcamera/control_ids.h> + +#include "libipa/histogram.h" +#include "libipa/lux.h" + +/** + * \file lux.h + */ + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +/** + * \class Lux + * \brief RkISP1 Lux control + * + * The Lux algorithm is responsible for estimating the lux level of the image. + * It doesn't take or generate any controls, but it provides a lux level for + * other algorithms (such as AGC) to use. + */ + +/** + * \brief Construct an rkisp1 Lux algo module + * + * The Lux helper is initialized to 65535 as that is the max bin count on the + * rkisp1. + */ +Lux::Lux() + : lux_(65535) +{ +} + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Lux::init([[maybe_unused]] IPAContext &context, const YamlObject &tuningData) +{ + return lux_.parseTuningData(tuningData); +} + +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void Lux::process(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats, + ControlList &metadata) +{ + utils::Duration exposureTime = context.configuration.sensor.lineDuration + * frameContext.sensor.exposure; + double gain = frameContext.sensor.gain; + + /* \todo Deduplicate the histogram calculation from AGC */ + const rkisp1_cif_isp_stat *params = &stats->params; + Histogram yHist({ params->hist.hist_bins, context.hw->numHistogramBins }, + [](uint32_t x) { return x >> 4; }); + + double lux = lux_.estimateLux(exposureTime, gain, 1.0, yHist); + frameContext.lux.lux = lux; + metadata.set(controls::Lux, lux); +} + +REGISTER_IPA_ALGORITHM(Lux, "Lux") + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/lux.h b/src/ipa/rkisp1/algorithms/lux.h new file mode 100644 index 00000000..8a90de55 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/lux.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * lux.h - RkISP1 Lux control + */ + +#pragma once + +#include <sys/types.h> + +#include "libipa/lux.h" + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +class Lux : public Algorithm +{ +public: + Lux(); + + int init(IPAContext &context, const YamlObject &tuningData) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats, + ControlList &metadata) override; + +private: + ipa::Lux lux_; +}; + +} /* namespace ipa::rkisp1::algorithms */ +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build index 1734a667..c66b0b70 100644 --- a/src/ipa/rkisp1/algorithms/meson.build +++ b/src/ipa/rkisp1/algorithms/meson.build @@ -12,4 +12,5 @@ rkisp1_ipa_algorithms = files([ 'goc.cpp', 'gsl.cpp', 'lsc.cpp', + 'lux.cpp', ]) diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp index 80b99df8..261c0472 100644 --- a/src/ipa/rkisp1/ipa_context.cpp +++ b/src/ipa/rkisp1/ipa_context.cpp @@ -165,8 +165,11 @@ namespace libcamera::ipa::rkisp1 { * \var IPAActiveState::agc.automatic.gain * \brief Automatic analogue gain multiplier * - * \var IPAActiveState::agc.autoEnabled - * \brief Manual/automatic AGC state as set by the AeEnable control + * \var IPAActiveState::agc.autoExposureEnabled + * \brief Manual/automatic AGC state (exposure) as set by the ExposureTimeMode control + * + * \var IPAActiveState::agc.autoGainEnabled + * \brief Manual/automatic AGC state (gain) as set by the AnalogueGainMode control * * \var IPAActiveState::agc.constraintMode * \brief Constraint mode as set by the AeConstraintMode control @@ -289,8 +292,11 @@ namespace libcamera::ipa::rkisp1 { * * The gain should be adapted to the sensor specific gain code before applying. * - * \var IPAFrameContext::agc.autoEnabled - * \brief Manual/automatic AGC state as set by the AeEnable control + * \var IPAFrameContext::agc.autoExposureEnabled + * \brief Manual/automatic AGC state (exposure) as set by the ExposureTimeMode control + * + * \var IPAFrameContext::agc.autoGainEnabled + * \brief Manual/automatic AGC state (gain) as set by the AnalogueGainMode control * * \var IPAFrameContext::agc.constraintMode * \brief Constraint mode as set by the AeConstraintMode control @@ -306,6 +312,16 @@ namespace libcamera::ipa::rkisp1 { * * \var IPAFrameContext::agc.updateMetering * \brief Indicate if new ISP AGC metering parameters need to be applied + * + * \var IPAFrameContext::agc.autoExposureModeChange + * \brief Indicate if autoExposureEnabled has changed from true in the previous + * frame to false in the current frame, and no manual exposure value has been + * supplied in the current frame. + * + * \var IPAFrameContext::agc.autoGainModeChange + * \brief Indicate if autoGainEnabled has changed from true in the previous + * frame to false in the current frame, and no manual gain value has been + * supplied in the current frame. */ /** diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h index deb8c196..c765b928 100644 --- a/src/ipa/rkisp1/ipa_context.h +++ b/src/ipa/rkisp1/ipa_context.h @@ -22,10 +22,10 @@ #include "libcamera/internal/debug_controls.h" #include "libcamera/internal/matrix.h" +#include "libcamera/internal/vector.h" #include <libipa/camera_sensor_helper.h> #include <libipa/fc_queue.h> -#include <libipa/vector.h> namespace libcamera { @@ -79,7 +79,8 @@ struct IPAActiveState { double gain; } automatic; - bool autoEnabled; + bool autoExposureEnabled; + bool autoGainEnabled; controls::AeConstraintModeEnum constraintMode; controls::AeExposureModeEnum exposureMode; controls::AeMeteringModeEnum meteringMode; @@ -124,17 +125,21 @@ struct IPAFrameContext : public FrameContext { struct { uint32_t exposure; double gain; - bool autoEnabled; + bool autoExposureEnabled; + bool autoGainEnabled; controls::AeConstraintModeEnum constraintMode; controls::AeExposureModeEnum exposureMode; controls::AeMeteringModeEnum meteringMode; utils::Duration maxFrameDuration; bool updateMetering; + bool autoExposureModeChange; + bool autoGainModeChange; } agc; struct { RGB<double> gains; bool autoEnabled; + unsigned int temperatureK; } awb; struct { @@ -168,6 +173,10 @@ struct IPAFrameContext : public FrameContext { struct { Matrix<float, 3, 3> ccm; } ccm; + + struct { + double lux; + } lux; }; struct IPAContext { diff --git a/src/ipa/rkisp1/meson.build b/src/ipa/rkisp1/meson.build index 34844f14..26a9fa40 100644 --- a/src/ipa/rkisp1/meson.build +++ b/src/ipa/rkisp1/meson.build @@ -9,7 +9,6 @@ rkisp1_ipa_sources = files([ 'ipa_context.cpp', 'params.cpp', 'rkisp1.cpp', - 'utils.cpp', ]) rkisp1_ipa_sources += rkisp1_ipa_algorithms diff --git a/src/ipa/rpi/cam_helper/cam_helper.cpp b/src/ipa/rpi/cam_helper/cam_helper.cpp index 6493e882..a78db9c1 100644 --- a/src/ipa/rpi/cam_helper/cam_helper.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper.cpp @@ -156,17 +156,9 @@ void CamHelper::setCameraMode(const CameraMode &mode) } } -void CamHelper::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const +void CamHelper::setHwConfig(const Controller::HardwareConfig &hwConfig) { - /* - * These values are correct for many sensors. Other sensors will - * need to over-ride this function. - */ - exposureDelay = 2; - gainDelay = 1; - vblankDelay = 2; - hblankDelay = 2; + hwConfig_ = hwConfig; } bool CamHelper::sensorEmbeddedDataPresent() const diff --git a/src/ipa/rpi/cam_helper/cam_helper.h b/src/ipa/rpi/cam_helper/cam_helper.h index 4a4ab5e6..4a826690 100644 --- a/src/ipa/rpi/cam_helper/cam_helper.h +++ b/src/ipa/rpi/cam_helper/cam_helper.h @@ -36,11 +36,6 @@ namespace RPiController { * exposure time, and to convert between the sensor's gain codes and actual * gains. * - * A function to return the number of frames of delay between updating exposure, - * analogue gain and vblanking, and for the changes to take effect. For many - * sensors these take the values 2, 1 and 2 respectively, but sensors that are - * different will need to over-ride the default function provided. - * * A function to query if the sensor outputs embedded data that can be parsed. * * A function to return the sensitivity of a given camera mode. @@ -76,6 +71,7 @@ public: CamHelper(std::unique_ptr<MdParser> parser, unsigned int frameIntegrationDiff); virtual ~CamHelper(); void setCameraMode(const CameraMode &mode); + void setHwConfig(const Controller::HardwareConfig &hwConfig); virtual void prepare(libcamera::Span<const uint8_t> buffer, Metadata &metadata); virtual void process(StatisticsPtr &stats, Metadata &metadata); @@ -91,8 +87,6 @@ public: libcamera::utils::Duration lineLengthPckToDuration(uint32_t lineLengthPck) const; virtual uint32_t gainCode(double gain) const = 0; virtual double gain(uint32_t gainCode) const = 0; - virtual void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const; virtual bool sensorEmbeddedDataPresent() const; virtual double getModeSensitivity(const CameraMode &mode) const; virtual unsigned int hideFramesStartup() const; @@ -108,6 +102,7 @@ protected: std::unique_ptr<MdParser> parser_; CameraMode mode_; + Controller::HardwareConfig hwConfig_; private: /* diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx283.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx283.cpp index cb0be72a..efc03193 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_imx283.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_imx283.cpp @@ -17,8 +17,6 @@ public: CamHelperImx283(); uint32_t gainCode(double gain) const override; double gain(uint32_t gainCode) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; unsigned int hideFramesModeSwitch() const override; private: @@ -49,16 +47,6 @@ double CamHelperImx283::gain(uint32_t gainCode) const return static_cast<double>(2048.0 / (2048 - gainCode)); } -void CamHelperImx283::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - /* The driver appears to behave as follows: */ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 2; - hblankDelay = 2; -} - unsigned int CamHelperImx283::hideFramesModeSwitch() const { /* After a mode switch, we seem to get 1 bad frame. */ diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx290.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx290.cpp index 3b87751e..c1aa8528 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_imx290.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_imx290.cpp @@ -17,8 +17,6 @@ public: CamHelperImx290(); uint32_t gainCode(double gain) const override; double gain(uint32_t gainCode) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; unsigned int hideFramesStartup() const override; unsigned int hideFramesModeSwitch() const override; @@ -46,15 +44,6 @@ double CamHelperImx290::gain(uint32_t gainCode) const return std::pow(10, 0.015 * gainCode); } -void CamHelperImx290::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 2; - hblankDelay = 2; -} - unsigned int CamHelperImx290::hideFramesStartup() const { /* On startup, we seem to get 1 bad frame. */ diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx296.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx296.cpp index d4a4fa79..ac7ee2ea 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_imx296.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_imx296.cpp @@ -23,8 +23,6 @@ public: double gain(uint32_t gainCode) const override; uint32_t exposureLines(const Duration exposure, const Duration lineLength) const override; Duration exposure(uint32_t exposureLines, const Duration lineLength) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; private: static constexpr uint32_t minExposureLines = 1; @@ -66,15 +64,6 @@ Duration CamHelperImx296::exposure(uint32_t exposureLines, return std::max<uint32_t>(minExposureLines, exposureLines) * timePerLine + 14.26us; } -void CamHelperImx296::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 2; - hblankDelay = 2; -} - static CamHelper *create() { return new CamHelperImx296(); diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx415.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx415.cpp new file mode 100644 index 00000000..c0a09eee --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper_imx415.cpp @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2025, Raspberry Pi Ltd + * + * camera helper for imx415 sensor + */ + +#include <cmath> + +#include "cam_helper.h" + +using namespace RPiController; + +class CamHelperImx415 : public CamHelper +{ +public: + CamHelperImx415(); + uint32_t gainCode(double gain) const override; + double gain(uint32_t gainCode) const override; + unsigned int hideFramesStartup() const override; + unsigned int hideFramesModeSwitch() const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 8; +}; + +CamHelperImx415::CamHelperImx415() + : CamHelper({}, frameIntegrationDiff) +{ +} + +uint32_t CamHelperImx415::gainCode(double gain) const +{ + int code = 66.6667 * std::log10(gain); + return std::max(0, std::min(code, 0xf0)); +} + +double CamHelperImx415::gain(uint32_t gainCode) const +{ + return std::pow(10, 0.015 * gainCode); +} + +unsigned int CamHelperImx415::hideFramesStartup() const +{ + /* On startup, we seem to get 1 bad frame. */ + return 1; +} + +unsigned int CamHelperImx415::hideFramesModeSwitch() const +{ + /* After a mode switch, we seem to get 1 bad frame. */ + return 1; +} + +static CamHelper *create() +{ + return new CamHelperImx415(); +} + +static RegisterCamHelper reg("imx415", &create); diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx477.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx477.cpp index a53c40cd..a72ac67d 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_imx477.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_imx477.cpp @@ -51,8 +51,6 @@ public: void prepare(libcamera::Span<const uint8_t> buffer, Metadata &metadata) override; std::pair<uint32_t, uint32_t> getBlanking(Duration &exposure, Duration minFrameDuration, Duration maxFrameDuration) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; bool sensorEmbeddedDataPresent() const override; private: @@ -159,15 +157,6 @@ std::pair<uint32_t, uint32_t> CamHelperImx477::getBlanking(Duration &exposure, return { frameLength - mode_.height, hblank }; } -void CamHelperImx477::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 3; - hblankDelay = 3; -} - bool CamHelperImx477::sensorEmbeddedDataPresent() const { return true; diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx519.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx519.cpp index 2ff08653..10cbea48 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_imx519.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_imx519.cpp @@ -51,8 +51,6 @@ public: void prepare(libcamera::Span<const uint8_t> buffer, Metadata &metadata) override; std::pair<uint32_t, uint32_t> getBlanking(Duration &exposure, Duration minFrameDuration, Duration maxFrameDuration) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; bool sensorEmbeddedDataPresent() const override; private: @@ -159,15 +157,6 @@ std::pair<uint32_t, uint32_t> CamHelperImx519::getBlanking(Duration &exposure, return { frameLength - mode_.height, hblank }; } -void CamHelperImx519::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 3; - hblankDelay = 3; -} - bool CamHelperImx519::sensorEmbeddedDataPresent() const { return true; diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx708.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx708.cpp index ec83d9fd..6150909c 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_imx708.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_imx708.cpp @@ -54,8 +54,6 @@ public: void process(StatisticsPtr &stats, Metadata &metadata) override; std::pair<uint32_t, uint32_t> getBlanking(Duration &exposure, Duration minFrameDuration, Duration maxFrameDuration) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; bool sensorEmbeddedDataPresent() const override; double getModeSensitivity(const CameraMode &mode) const override; unsigned int hideFramesModeSwitch() const override; @@ -66,7 +64,7 @@ private: * Smallest difference between the frame length and integration time, * in units of lines. */ - static constexpr int frameIntegrationDiff = 22; + static constexpr int frameIntegrationDiff = 48; /* Maximum frame length allowable for long exposure calculations. */ static constexpr int frameLengthMax = 0xffdc; /* Largest long exposure scale factor given as a left shift on the frame length. */ @@ -208,15 +206,6 @@ std::pair<uint32_t, uint32_t> CamHelperImx708::getBlanking(Duration &exposure, return { frameLength - mode_.height, hblank }; } -void CamHelperImx708::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 3; - hblankDelay = 3; -} - bool CamHelperImx708::sensorEmbeddedDataPresent() const { return true; diff --git a/src/ipa/rpi/cam_helper/cam_helper_ov5647.cpp b/src/ipa/rpi/cam_helper/cam_helper_ov5647.cpp index c30b017c..40d6b6d7 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_ov5647.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_ov5647.cpp @@ -17,8 +17,6 @@ public: CamHelperOv5647(); uint32_t gainCode(double gain) const override; double gain(uint32_t gainCode) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; unsigned int hideFramesStartup() const override; unsigned int hideFramesModeSwitch() const override; unsigned int mistrustFramesStartup() const override; @@ -52,19 +50,6 @@ double CamHelperOv5647::gain(uint32_t gainCode) const return static_cast<double>(gainCode) / 16.0; } -void CamHelperOv5647::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - /* - * We run this sensor in a mode where the gain delay is bumped up to - * 2. It seems to be the only way to make the delays "predictable". - */ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 2; - hblankDelay = 2; -} - unsigned int CamHelperOv5647::hideFramesStartup() const { /* diff --git a/src/ipa/rpi/cam_helper/cam_helper_ov64a40.cpp b/src/ipa/rpi/cam_helper/cam_helper_ov64a40.cpp index a8efd389..980495a8 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_ov64a40.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_ov64a40.cpp @@ -18,8 +18,6 @@ public: CamHelperOv64a40(); uint32_t gainCode(double gain) const override; double gain(uint32_t gainCode) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; double getModeSensitivity(const CameraMode &mode) const override; private: @@ -45,16 +43,6 @@ double CamHelperOv64a40::gain(uint32_t gainCode) const return static_cast<double>(gainCode) / 128.0; } -void CamHelperOv64a40::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - /* The driver appears to behave as follows: */ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 2; - hblankDelay = 2; -} - double CamHelperOv64a40::getModeSensitivity(const CameraMode &mode) const { if (mode.binX >= 2 && mode.scaleX >= 4) { diff --git a/src/ipa/rpi/cam_helper/cam_helper_ov7251.cpp b/src/ipa/rpi/cam_helper/cam_helper_ov7251.cpp index 7b12c445..fc7b999f 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_ov7251.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_ov7251.cpp @@ -17,8 +17,6 @@ public: CamHelperOv7251(); uint32_t gainCode(double gain) const override; double gain(uint32_t gainCode) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; private: /* @@ -48,16 +46,6 @@ double CamHelperOv7251::gain(uint32_t gainCode) const return static_cast<double>(gainCode) / 16.0; } -void CamHelperOv7251::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - /* The driver appears to behave as follows: */ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 2; - hblankDelay = 2; -} - static CamHelper *create() { return new CamHelperOv7251(); diff --git a/src/ipa/rpi/cam_helper/cam_helper_ov9281.cpp b/src/ipa/rpi/cam_helper/cam_helper_ov9281.cpp index a65c8ac0..e93a4691 100644 --- a/src/ipa/rpi/cam_helper/cam_helper_ov9281.cpp +++ b/src/ipa/rpi/cam_helper/cam_helper_ov9281.cpp @@ -17,15 +17,13 @@ public: CamHelperOv9281(); uint32_t gainCode(double gain) const override; double gain(uint32_t gainCode) const override; - void getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const override; private: /* * Smallest difference between the frame length and integration time, * in units of lines. */ - static constexpr int frameIntegrationDiff = 4; + static constexpr int frameIntegrationDiff = 25; }; /* @@ -48,16 +46,6 @@ double CamHelperOv9281::gain(uint32_t gainCode) const return static_cast<double>(gainCode) / 16.0; } -void CamHelperOv9281::getDelays(int &exposureDelay, int &gainDelay, - int &vblankDelay, int &hblankDelay) const -{ - /* The driver appears to behave as follows: */ - exposureDelay = 2; - gainDelay = 2; - vblankDelay = 2; - hblankDelay = 2; -} - static CamHelper *create() { return new CamHelperOv9281(); diff --git a/src/ipa/rpi/cam_helper/meson.build b/src/ipa/rpi/cam_helper/meson.build index 03e88fe0..abf02147 100644 --- a/src/ipa/rpi/cam_helper/meson.build +++ b/src/ipa/rpi/cam_helper/meson.build @@ -7,6 +7,7 @@ rpi_ipa_cam_helper_sources = files([ 'cam_helper_imx283.cpp', 'cam_helper_imx290.cpp', 'cam_helper_imx296.cpp', + 'cam_helper_imx415.cpp', 'cam_helper_imx477.cpp', 'cam_helper_imx519.cpp', 'cam_helper_imx708.cpp', diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index 5fce17e6..6734c32e 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -55,9 +55,19 @@ constexpr Duration controllerMinFrameDuration = 1.0s / 30.0; /* List of controls handled by the Raspberry Pi IPA */ const ControlInfoMap::Map ipaControls{ - { &controls::AeEnable, ControlInfo(false, true) }, - { &controls::ExposureTime, ControlInfo(0, 66666) }, - { &controls::AnalogueGain, ControlInfo(1.0f, 16.0f) }, + /* \todo Move this to the Camera class */ + { &controls::AeEnable, ControlInfo(false, true, true) }, + { &controls::ExposureTimeMode, + ControlInfo(static_cast<int32_t>(controls::ExposureTimeModeAuto), + static_cast<int32_t>(controls::ExposureTimeModeManual), + static_cast<int32_t>(controls::ExposureTimeModeAuto)) }, + { &controls::ExposureTime, + ControlInfo(1, 66666, static_cast<int32_t>(defaultExposureTime.get<std::micro>())) }, + { &controls::AnalogueGainMode, + ControlInfo(static_cast<int32_t>(controls::AnalogueGainModeAuto), + static_cast<int32_t>(controls::AnalogueGainModeManual), + static_cast<int32_t>(controls::AnalogueGainModeAuto)) }, + { &controls::AnalogueGain, ControlInfo(1.0f, 16.0f, 1.0f) }, { &controls::AeMeteringMode, ControlInfo(controls::AeMeteringModeValues) }, { &controls::AeConstraintMode, ControlInfo(controls::AeConstraintModeValues) }, { &controls::AeExposureMode, ControlInfo(controls::AeExposureModeValues) }, @@ -71,7 +81,9 @@ const ControlInfoMap::Map ipaControls{ { &controls::HdrMode, ControlInfo(controls::HdrModeValues) }, { &controls::Sharpness, ControlInfo(0.0f, 16.0f, 1.0f) }, { &controls::ScalerCrop, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) }, - { &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) }, + { &controls::FrameDurationLimits, + ControlInfo(INT64_C(33333), INT64_C(120000), + static_cast<int64_t>(defaultMinFrameDuration.get<std::micro>())) }, { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) }, { &controls::rpi::StatsOutputEnable, ControlInfo(false, true, false) }, }; @@ -81,6 +93,7 @@ const ControlInfoMap::Map ipaColourControls{ { &controls::AwbEnable, ControlInfo(false, true) }, { &controls::AwbMode, ControlInfo(controls::AwbModeValues) }, { &controls::ColourGains, ControlInfo(0.0f, 32.0f) }, + { &controls::ColourTemperature, ControlInfo(100, 100000) }, { &controls::Saturation, ControlInfo(0.0f, 32.0f, 1.0f) }, }; @@ -134,18 +147,8 @@ int32_t IpaBase::init(const IPASettings &settings, const InitParams ¶ms, Ini return -EINVAL; } - /* - * Pass out the sensor config to the pipeline handler in order - * to setup the staggered writer class. - */ - int gainDelay, exposureDelay, vblankDelay, hblankDelay, sensorMetadata; - helper_->getDelays(exposureDelay, gainDelay, vblankDelay, hblankDelay); - sensorMetadata = helper_->sensorEmbeddedDataPresent(); - - result->sensorConfig.gainDelay = gainDelay; - result->sensorConfig.exposureDelay = exposureDelay; - result->sensorConfig.vblankDelay = vblankDelay; - result->sensorConfig.hblankDelay = hblankDelay; + /* Pass out the sensor metadata to the pipeline handler */ + int sensorMetadata = helper_->sensorEmbeddedDataPresent(); result->sensorConfig.sensorMetadata = sensorMetadata; /* Load the tuning file for this sensor. */ @@ -160,6 +163,7 @@ int32_t IpaBase::init(const IPASettings &settings, const InitParams ¶ms, Ini lensPresent_ = params.lensPresent; controller_.initialise(); + helper_->setHwConfig(controller_.getHardwareConfig()); /* Return the controls handled by the IPA */ ControlInfoMap::Map ctrlMap = ipaControls; @@ -258,15 +262,18 @@ int32_t IpaBase::configure(const IPACameraSensorInfo &sensorInfo, const ConfigPa ControlInfoMap::Map ctrlMap = ipaControls; ctrlMap[&controls::FrameDurationLimits] = ControlInfo(static_cast<int64_t>(mode_.minFrameDuration.get<std::micro>()), - static_cast<int64_t>(mode_.maxFrameDuration.get<std::micro>())); + static_cast<int64_t>(mode_.maxFrameDuration.get<std::micro>()), + static_cast<int64_t>(defaultMinFrameDuration.get<std::micro>())); ctrlMap[&controls::AnalogueGain] = ControlInfo(static_cast<float>(mode_.minAnalogueGain), - static_cast<float>(mode_.maxAnalogueGain)); + static_cast<float>(mode_.maxAnalogueGain), + static_cast<float>(defaultAnalogueGain)); ctrlMap[&controls::ExposureTime] = ControlInfo(static_cast<int32_t>(mode_.minExposureTime.get<std::micro>()), - static_cast<int32_t>(mode_.maxExposureTime.get<std::micro>())); + static_cast<int32_t>(mode_.maxExposureTime.get<std::micro>()), + static_cast<int32_t>(defaultExposureTime.get<std::micro>())); /* Declare colour processing related controls for non-mono sensors. */ if (!monoSensor_) @@ -757,6 +764,42 @@ void IpaBase::applyControls(const ControlList &controls) af->setMode(mode->second); } + /* + * Because some AE controls are mode-specific, handle the AE-related + * mode changes first. + */ + const auto analogueGainMode = controls.get(controls::AnalogueGainMode); + const auto exposureTimeMode = controls.get(controls::ExposureTimeMode); + + if (analogueGainMode || exposureTimeMode) { + RPiController::AgcAlgorithm *agc = dynamic_cast<RPiController::AgcAlgorithm *>( + controller_.getAlgorithm("agc")); + if (agc) { + if (analogueGainMode) { + if (*analogueGainMode == controls::AnalogueGainModeManual) + agc->disableAutoGain(); + else + agc->enableAutoGain(); + + libcameraMetadata_.set(controls::AnalogueGainMode, + *analogueGainMode); + } + + if (exposureTimeMode) { + if (*exposureTimeMode == controls::ExposureTimeModeManual) + agc->disableAutoExposure(); + else + agc->enableAutoExposure(); + + libcameraMetadata_.set(controls::ExposureTimeMode, + *exposureTimeMode); + } + } else { + LOG(IPARPI, Warning) + << "Could not set AnalogueGainMode or ExposureTimeMode - no AGC algorithm"; + } + } + /* Iterate over controls */ for (auto const &ctrl : controls) { LOG(IPARPI, Debug) << "Request ctrl: " @@ -764,23 +807,8 @@ void IpaBase::applyControls(const ControlList &controls) << " = " << ctrl.second.toString(); switch (ctrl.first) { - case controls::AE_ENABLE: { - RPiController::AgcAlgorithm *agc = dynamic_cast<RPiController::AgcAlgorithm *>( - controller_.getAlgorithm("agc")); - if (!agc) { - LOG(IPARPI, Warning) - << "Could not set AE_ENABLE - no AGC algorithm"; - break; - } - - if (ctrl.second.get<bool>() == false) - agc->disableAuto(); - else - agc->enableAuto(); - - libcameraMetadata_.set(controls::AeEnable, ctrl.second.get<bool>()); - break; - } + case controls::EXPOSURE_TIME_MODE: + break; /* We already handled this one above */ case controls::EXPOSURE_TIME: { RPiController::AgcAlgorithm *agc = dynamic_cast<RPiController::AgcAlgorithm *>( @@ -791,6 +819,13 @@ void IpaBase::applyControls(const ControlList &controls) break; } + /* + * Ignore manual exposure time when the auto exposure + * algorithm is running. + */ + if (agc->autoExposureEnabled()) + break; + /* The control provides units of microseconds. */ agc->setFixedExposureTime(0, ctrl.second.get<int32_t>() * 1.0us); @@ -798,6 +833,9 @@ void IpaBase::applyControls(const ControlList &controls) break; } + case controls::ANALOGUE_GAIN_MODE: + break; /* We already handled this one above */ + case controls::ANALOGUE_GAIN: { RPiController::AgcAlgorithm *agc = dynamic_cast<RPiController::AgcAlgorithm *>( controller_.getAlgorithm("agc")); @@ -807,6 +845,13 @@ void IpaBase::applyControls(const ControlList &controls) break; } + /* + * Ignore manual analogue gain value when the auto gain + * algorithm is running. + */ + if (agc->autoGainEnabled()) + break; + agc->setFixedAnalogueGain(0, ctrl.second.get<float>()); libcameraMetadata_.set(controls::AnalogueGain, @@ -863,6 +908,13 @@ void IpaBase::applyControls(const ControlList &controls) break; } + /* + * Ignore AE_EXPOSURE_MODE if the shutter or the gain + * are in auto mode. + */ + if (agc->autoExposureEnabled() || agc->autoGainEnabled()) + break; + int32_t idx = ctrl.second.get<int32_t>(); if (ExposureModeTable.count(idx)) { agc->setExposureMode(ExposureModeTable.at(idx)); @@ -1021,6 +1073,25 @@ void IpaBase::applyControls(const ControlList &controls) break; } + case controls::COLOUR_TEMPERATURE: { + /* Silently ignore this control for a mono sensor. */ + if (monoSensor_) + break; + + auto temperatureK = ctrl.second.get<int32_t>(); + RPiController::AwbAlgorithm *awb = dynamic_cast<RPiController::AwbAlgorithm *>( + controller_.getAlgorithm("awb")); + if (!awb) { + LOG(IPARPI, Warning) + << "Could not set COLOUR_TEMPERATURE - no AWB algorithm"; + break; + } + + awb->setColourTemperature(temperatureK); + /* This metadata will get reported back automatically. */ + break; + } + case controls::BRIGHTNESS: { RPiController::ContrastAlgorithm *contrast = dynamic_cast<RPiController::ContrastAlgorithm *>( controller_.getAlgorithm("contrast")); @@ -1323,9 +1394,19 @@ void IpaBase::reportMetadata(unsigned int ipaContext) } AgcPrepareStatus *agcPrepareStatus = rpiMetadata.getLocked<AgcPrepareStatus>("agc.prepare_status"); - if (agcPrepareStatus) { - libcameraMetadata_.set(controls::AeLocked, agcPrepareStatus->locked); + if (agcPrepareStatus) libcameraMetadata_.set(controls::DigitalGain, agcPrepareStatus->digitalGain); + + RPiController::AgcAlgorithm *agc = dynamic_cast<RPiController::AgcAlgorithm *>( + controller_.getAlgorithm("agc")); + if (agc) { + if (!agc->autoExposureEnabled() && !agc->autoGainEnabled()) + libcameraMetadata_.set(controls::AeState, controls::AeStateIdle); + else if (agcPrepareStatus) + libcameraMetadata_.set(controls::AeState, + agcPrepareStatus->locked + ? controls::AeStateConverged + : controls::AeStateSearching); } LuxStatus *luxStatus = rpiMetadata.getLocked<LuxStatus>("lux.status"); diff --git a/src/ipa/rpi/controller/agc_algorithm.h b/src/ipa/rpi/controller/agc_algorithm.h index c9782857..fdaa10e6 100644 --- a/src/ipa/rpi/controller/agc_algorithm.h +++ b/src/ipa/rpi/controller/agc_algorithm.h @@ -30,8 +30,12 @@ public: virtual void setMeteringMode(std::string const &meteringModeName) = 0; virtual void setExposureMode(std::string const &exposureModeName) = 0; virtual void setConstraintMode(std::string const &contraintModeName) = 0; - virtual void enableAuto() = 0; - virtual void disableAuto() = 0; + virtual void enableAutoExposure() = 0; + virtual void disableAutoExposure() = 0; + virtual bool autoExposureEnabled() const = 0; + virtual void enableAutoGain() = 0; + virtual void disableAutoGain() = 0; + virtual bool autoGainEnabled() const = 0; virtual void setActiveChannels(const std::vector<unsigned int> &activeChannels) = 0; }; diff --git a/src/ipa/rpi/controller/awb_algorithm.h b/src/ipa/rpi/controller/awb_algorithm.h index 1779b050..d941ed4e 100644 --- a/src/ipa/rpi/controller/awb_algorithm.h +++ b/src/ipa/rpi/controller/awb_algorithm.h @@ -19,6 +19,7 @@ public: virtual void initialValues(double &gainR, double &gainB) = 0; virtual void setMode(std::string const &modeName) = 0; virtual void setManualGains(double manualR, double manualB) = 0; + virtual void setColourTemperature(double temperatureK) = 0; virtual void enableAuto() = 0; virtual void disableAuto() = 0; }; diff --git a/src/ipa/rpi/controller/controller.cpp b/src/ipa/rpi/controller/controller.cpp index e0131018..651fff63 100644 --- a/src/ipa/rpi/controller/controller.cpp +++ b/src/ipa/rpi/controller/controller.cpp @@ -39,6 +39,7 @@ static const std::map<std::string, Controller::HardwareConfig> HardwareConfigMap .pipelineWidth = 13, .statsInline = false, .minPixelProcessingTime = 0s, + .dataBufferStrided = true, } }, { @@ -71,6 +72,7 @@ static const std::map<std::string, Controller::HardwareConfig> HardwareConfigMap * frames wider than ~16,000 pixels. */ .minPixelProcessingTime = 1.0us / 380, + .dataBufferStrided = false, } }, }; diff --git a/src/ipa/rpi/controller/controller.h b/src/ipa/rpi/controller/controller.h index eff520bd..fdb46557 100644 --- a/src/ipa/rpi/controller/controller.h +++ b/src/ipa/rpi/controller/controller.h @@ -49,6 +49,7 @@ public: unsigned int pipelineWidth; bool statsInline; libcamera::utils::Duration minPixelProcessingTime; + bool dataBufferStrided; }; Controller(); diff --git a/src/ipa/rpi/controller/metadata.h b/src/ipa/rpi/controller/metadata.h index b4650d25..77d3b074 100644 --- a/src/ipa/rpi/controller/metadata.h +++ b/src/ipa/rpi/controller/metadata.h @@ -12,6 +12,7 @@ #include <map> #include <mutex> #include <string> +#include <utility> #include <libcamera/base/thread_annotations.h> @@ -36,10 +37,10 @@ public: } template<typename T> - void set(std::string const &tag, T const &value) + void set(std::string const &tag, T &&value) { std::scoped_lock lock(mutex_); - data_[tag] = value; + data_[tag] = std::forward<T>(value); } template<typename T> @@ -90,6 +91,12 @@ public: data_.insert(other.data_.begin(), other.data_.end()); } + void erase(std::string const &tag) + { + std::scoped_lock lock(mutex_); + eraseLocked(tag); + } + template<typename T> T *getLocked(std::string const &tag) { @@ -104,10 +111,18 @@ public: } template<typename T> - void setLocked(std::string const &tag, T const &value) + void setLocked(std::string const &tag, T &&value) { /* Use this only if you're holding the lock yourself. */ - data_[tag] = value; + data_[tag] = std::forward<T>(value); + } + + void eraseLocked(std::string const &tag) + { + auto it = data_.find(tag); + if (it == data_.end()) + return; + data_.erase(it); } /* diff --git a/src/ipa/rpi/controller/rpi/agc.cpp b/src/ipa/rpi/controller/rpi/agc.cpp index c48fdf15..02bfdb4a 100644 --- a/src/ipa/rpi/controller/rpi/agc.cpp +++ b/src/ipa/rpi/controller/rpi/agc.cpp @@ -74,22 +74,62 @@ int Agc::checkChannel(unsigned int channelIndex) const return 0; } -void Agc::disableAuto() +void Agc::disableAutoExposure() { - LOG(RPiAgc, Debug) << "disableAuto"; + LOG(RPiAgc, Debug) << "disableAutoExposure"; /* All channels are enabled/disabled together. */ for (auto &data : channelData_) - data.channel.disableAuto(); + data.channel.disableAutoExposure(); } -void Agc::enableAuto() +void Agc::enableAutoExposure() { - LOG(RPiAgc, Debug) << "enableAuto"; + LOG(RPiAgc, Debug) << "enableAutoExposure"; /* All channels are enabled/disabled together. */ for (auto &data : channelData_) - data.channel.enableAuto(); + data.channel.enableAutoExposure(); +} + +bool Agc::autoExposureEnabled() const +{ + LOG(RPiAgc, Debug) << "autoExposureEnabled"; + + /* + * We always have at least one channel, and since all channels are + * enabled and disabled together we can simply check the first one. + */ + return channelData_[0].channel.autoExposureEnabled(); +} + +void Agc::disableAutoGain() +{ + LOG(RPiAgc, Debug) << "disableAutoGain"; + + /* All channels are enabled/disabled together. */ + for (auto &data : channelData_) + data.channel.disableAutoGain(); +} + +void Agc::enableAutoGain() +{ + LOG(RPiAgc, Debug) << "enableAutoGain"; + + /* All channels are enabled/disabled together. */ + for (auto &data : channelData_) + data.channel.enableAutoGain(); +} + +bool Agc::autoGainEnabled() const +{ + LOG(RPiAgc, Debug) << "autoGainEnabled"; + + /* + * We always have at least one channel, and since all channels are + * enabled and disabled together we can simply check the first one. + */ + return channelData_[0].channel.autoGainEnabled(); } unsigned int Agc::getConvergenceFrames() const diff --git a/src/ipa/rpi/controller/rpi/agc.h b/src/ipa/rpi/controller/rpi/agc.h index 3aca000b..c3a940bf 100644 --- a/src/ipa/rpi/controller/rpi/agc.h +++ b/src/ipa/rpi/controller/rpi/agc.h @@ -40,8 +40,12 @@ public: void setMeteringMode(std::string const &meteringModeName) override; void setExposureMode(std::string const &exposureModeName) override; void setConstraintMode(std::string const &contraintModeName) override; - void enableAuto() override; - void disableAuto() override; + void enableAutoExposure() override; + void disableAutoExposure() override; + bool autoExposureEnabled() const override; + void enableAutoGain() override; + void disableAutoGain() override; + bool autoGainEnabled() const override; void switchMode(CameraMode const &cameraMode, Metadata *metadata) override; void prepare(Metadata *imageMetadata) override; void process(StatisticsPtr &stats, Metadata *imageMetadata) override; diff --git a/src/ipa/rpi/controller/rpi/agc_channel.cpp b/src/ipa/rpi/controller/rpi/agc_channel.cpp index 79c45973..a5562760 100644 --- a/src/ipa/rpi/controller/rpi/agc_channel.cpp +++ b/src/ipa/rpi/controller/rpi/agc_channel.cpp @@ -12,8 +12,9 @@ #include <libcamera/base/log.h> +#include "libcamera/internal/vector.h" + #include "libipa/colours.h" -#include "libipa/vector.h" #include "../awb_status.h" #include "../device_status.h" @@ -319,18 +320,36 @@ int AgcChannel::read(const libcamera::YamlObject ¶ms, return 0; } -void AgcChannel::disableAuto() +void AgcChannel::disableAutoExposure() { fixedExposureTime_ = status_.exposureTime; - fixedAnalogueGain_ = status_.analogueGain; } -void AgcChannel::enableAuto() +void AgcChannel::enableAutoExposure() { fixedExposureTime_ = 0s; +} + +bool AgcChannel::autoExposureEnabled() const +{ + return fixedExposureTime_ == 0s; +} + +void AgcChannel::disableAutoGain() +{ + fixedAnalogueGain_ = status_.analogueGain; +} + +void AgcChannel::enableAutoGain() +{ fixedAnalogueGain_ = 0; } +bool AgcChannel::autoGainEnabled() const +{ + return fixedAnalogueGain_ == 0; +} + unsigned int AgcChannel::getConvergenceFrames() const { /* @@ -682,7 +701,7 @@ static double computeInitialY(StatisticsPtr &stats, AwbStatus const &awb, * Note that the weights are applied by the IPA to the statistics directly, * before they are given to us here. */ - ipa::RGB<double> sum{ 0.0 }; + RGB<double> sum{ 0.0 }; double pixelSum = 0; for (unsigned int i = 0; i < stats->agcRegions.numRegions(); i++) { auto ®ion = stats->agcRegions.get(i); @@ -698,7 +717,7 @@ static double computeInitialY(StatisticsPtr &stats, AwbStatus const &awb, /* Factor in the AWB correction if needed. */ if (stats->agcStatsPos == Statistics::AgcStatsPos::PreWb) - sum *= ipa::RGB<double>{{ awb.gainR, awb.gainR, awb.gainB }}; + sum *= RGB<double>{ { awb.gainR, awb.gainR, awb.gainB } }; double ySum = ipa::rec601LuminanceFromRGB(sum); diff --git a/src/ipa/rpi/controller/rpi/agc_channel.h b/src/ipa/rpi/controller/rpi/agc_channel.h index 734e5efd..fa697e6f 100644 --- a/src/ipa/rpi/controller/rpi/agc_channel.h +++ b/src/ipa/rpi/controller/rpi/agc_channel.h @@ -96,8 +96,12 @@ public: void setMeteringMode(std::string const &meteringModeName); void setExposureMode(std::string const &exposureModeName); void setConstraintMode(std::string const &contraintModeName); - void enableAuto(); - void disableAuto(); + void enableAutoExposure(); + void disableAutoExposure(); + bool autoExposureEnabled() const; + void enableAutoGain(); + void disableAutoGain(); + bool autoGainEnabled() const; void switchMode(CameraMode const &cameraMode, Metadata *metadata); void prepare(Metadata *imageMetadata); void process(StatisticsPtr &stats, DeviceStatus const &deviceStatus, Metadata *imageMetadata, diff --git a/src/ipa/rpi/controller/rpi/awb.cpp b/src/ipa/rpi/controller/rpi/awb.cpp index c277a176..8479ae40 100644 --- a/src/ipa/rpi/controller/rpi/awb.cpp +++ b/src/ipa/rpi/controller/rpi/awb.cpp @@ -293,6 +293,24 @@ void Awb::setManualGains(double manualR, double manualB) } } +void Awb::setColourTemperature(double temperatureK) +{ + if (!config_.bayes) { + LOG(RPiAwb, Warning) << "AWB uncalibrated - cannot set colour temperature"; + return; + } + + temperatureK = config_.ctR.domain().clamp(temperatureK); + manualR_ = 1 / config_.ctR.eval(temperatureK); + manualB_ = 1 / config_.ctB.eval(temperatureK); + + syncResults_.temperatureK = temperatureK; + syncResults_.gainR = manualR_; + syncResults_.gainG = 1.0; + syncResults_.gainB = manualB_; + prevSyncResults_ = syncResults_; +} + void Awb::switchMode([[maybe_unused]] CameraMode const &cameraMode, Metadata *metadata) { diff --git a/src/ipa/rpi/controller/rpi/awb.h b/src/ipa/rpi/controller/rpi/awb.h index 5d628b47..86640f8f 100644 --- a/src/ipa/rpi/controller/rpi/awb.h +++ b/src/ipa/rpi/controller/rpi/awb.h @@ -105,6 +105,7 @@ public: void initialValues(double &gainR, double &gainB) override; void setMode(std::string const &name) override; void setManualGains(double manualR, double manualB) override; + void setColourTemperature(double temperatureK) override; void enableAuto() override; void disableAuto() override; void switchMode(CameraMode const &cameraMode, Metadata *metadata) override; diff --git a/src/ipa/rpi/vc4/data/imx415.json b/src/ipa/rpi/vc4/data/imx415.json new file mode 100755 index 00000000..6ed16b17 --- /dev/null +++ b/src/ipa/rpi/vc4/data/imx415.json @@ -0,0 +1,413 @@ +{ + "version": 2.0, + "target": "bcm2835", + "algorithms": [ + { + "rpi.black_level": + { + "black_level": 3840 + } + }, + { + "rpi.dpc": { } + }, + { + "rpi.lux": + { + "reference_shutter_speed": 19230, + "reference_gain": 1.0, + "reference_aperture": 1.0, + "reference_lux": 1198, + "reference_Y": 14876 + } + }, + { + "rpi.noise": + { + "reference_constant": 17, + "reference_slope": 3.439 + } + }, + { + "rpi.geq": + { + "offset": 193, + "slope": 0.00902 + } + }, + { + "rpi.sdn": { } + }, + { + "rpi.awb": + { + "priors": [ + { + "lux": 0, + "prior": + [ + 2000, 1.0, + 3000, 0.0, + 13000, 0.0 + ] + }, + { + "lux": 800, + "prior": + [ + 2000, 0.0, + 6000, 2.0, + 13000, 2.0 + ] + }, + { + "lux": 1500, + "prior": + [ + 2000, 0.0, + 4000, 1.0, + 6000, 6.0, + 6500, 7.0, + 7000, 1.0, + 13000, 1.0 + ] + } + ], + "modes": + { + "auto": + { + "lo": 2500, + "hi": 8000 + }, + "incandescent": + { + "lo": 2500, + "hi": 3000 + }, + "tungsten": + { + "lo": 3000, + "hi": 3500 + }, + "fluorescent": + { + "lo": 4000, + "hi": 4700 + }, + "indoor": + { + "lo": 3000, + "hi": 5000 + }, + "daylight": + { + "lo": 5500, + "hi": 6500 + }, + "cloudy": + { + "lo": 7000, + "hi": 8600 + } + }, + "bayes": 1, + "ct_curve": + [ + 2698.0, 0.7681, 0.2026, + 2930.0, 0.7515, 0.2116, + 3643.0, 0.6355, 0.2858, + 4605.0, 0.4992, 0.4041, + 5658.0, 0.4498, 0.4574 + ], + "sensitivity_r": 1.0, + "sensitivity_b": 1.0, + "transverse_pos": 0.0112, + "transverse_neg": 0.01424 + } + }, + { + "rpi.agc": + { + "metering_modes": + { + "centre-weighted": + { + "weights": + [ + 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0 + ] + }, + "spot": + { + "weights": + [ + 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + ] + }, + "matrix": + { + "weights": + [ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + ] + } + }, + "exposure_modes": + { + "normal": + { + "shutter": [ 100, 10000, 30000, 60000, 120000 ], + "gain": [ 1.0, 2.0, 4.0, 6.0, 6.0 ] + }, + "short": + { + "shutter": [ 100, 5000, 10000, 20000, 120000 ], + "gain": [ 1.0, 2.0, 4.0, 6.0, 6.0 ] + } + }, + "constraint_modes": + { + "normal": [ + { + "bound": "LOWER", + "q_lo": 0.98, + "q_hi": 1.0, + "y_target": + [ + 0, 0.5, + 1000, 0.5 + ] + } + ], + "highlight": [ + { + "bound": "LOWER", + "q_lo": 0.98, + "q_hi": 1.0, + "y_target": + [ + 0, 0.5, + 1000, 0.5 + ] + }, + { + "bound": "UPPER", + "q_lo": 0.98, + "q_hi": 1.0, + "y_target": + [ + 0, 0.8, + 1000, 0.8 + ] + } + ] + }, + "y_target": + [ + 0, 0.16, + 1000, 0.165, + 10000, 0.17 + ] + } + }, + { + "rpi.alsc": + { + "omega": 1.3, + "n_iter": 100, + "luminance_strength": 0.8, + "calibrations_Cr": [ + { + "ct": 3000, + "table": + [ + 1.025, 1.016, 1.013, 1.011, 1.008, 1.005, 1.003, 1.001, 1.003, 1.005, 1.008, 1.011, 1.014, 1.019, 1.027, 1.035, + 1.025, 1.017, 1.013, 1.011, 1.008, 1.005, 1.003, 1.003, 1.004, 1.005, 1.009, 1.012, 1.017, 1.023, 1.029, 1.035, + 1.022, 1.017, 1.013, 1.009, 1.007, 1.005, 1.003, 1.003, 1.004, 1.006, 1.009, 1.012, 1.017, 1.023, 1.029, 1.035, + 1.019, 1.015, 1.011, 1.007, 1.005, 1.003, 1.001, 1.001, 1.003, 1.004, 1.007, 1.009, 1.015, 1.022, 1.028, 1.035, + 1.018, 1.014, 1.009, 1.006, 1.004, 1.002, 1.001, 1.001, 1.001, 1.003, 1.006, 1.009, 1.015, 1.021, 1.028, 1.035, + 1.018, 1.013, 1.011, 1.006, 1.003, 1.002, 1.001, 1.001, 1.001, 1.003, 1.006, 1.009, 1.015, 1.022, 1.028, 1.036, + 1.018, 1.014, 1.011, 1.007, 1.004, 1.002, 1.001, 1.001, 1.001, 1.004, 1.007, 1.009, 1.015, 1.023, 1.029, 1.036, + 1.019, 1.014, 1.012, 1.008, 1.005, 1.003, 1.002, 1.001, 1.003, 1.005, 1.008, 1.012, 1.016, 1.024, 1.031, 1.037, + 1.021, 1.016, 1.013, 1.009, 1.008, 1.005, 1.003, 1.003, 1.005, 1.008, 1.011, 1.014, 1.019, 1.026, 1.033, 1.039, + 1.025, 1.021, 1.016, 1.013, 1.009, 1.008, 1.006, 1.006, 1.008, 1.011, 1.014, 1.019, 1.024, 1.031, 1.038, 1.046, + 1.029, 1.025, 1.021, 1.018, 1.014, 1.013, 1.011, 1.011, 1.012, 1.015, 1.019, 1.023, 1.028, 1.035, 1.046, 1.051, + 1.032, 1.029, 1.023, 1.021, 1.018, 1.015, 1.014, 1.014, 1.015, 1.018, 1.022, 1.027, 1.033, 1.041, 1.051, 1.054 + ] + }, + { + "ct": 5000, + "table": + [ + 1.025, 1.011, 1.009, 1.005, 1.004, 1.003, 1.001, 1.001, 1.002, 1.006, 1.009, 1.012, 1.016, 1.021, 1.031, 1.041, + 1.025, 1.014, 1.009, 1.007, 1.005, 1.004, 1.003, 1.003, 1.004, 1.007, 1.009, 1.013, 1.021, 1.028, 1.037, 1.041, + 1.023, 1.014, 1.009, 1.007, 1.005, 1.004, 1.003, 1.003, 1.005, 1.007, 1.011, 1.014, 1.021, 1.028, 1.037, 1.048, + 1.022, 1.012, 1.007, 1.005, 1.002, 1.001, 1.001, 1.001, 1.003, 1.005, 1.009, 1.014, 1.019, 1.028, 1.039, 1.048, + 1.022, 1.011, 1.006, 1.003, 1.001, 1.001, 1.001, 1.001, 1.002, 1.005, 1.009, 1.014, 1.021, 1.029, 1.039, 1.051, + 1.022, 1.012, 1.007, 1.003, 1.002, 1.001, 1.001, 1.001, 1.002, 1.005, 1.009, 1.015, 1.021, 1.031, 1.041, 1.053, + 1.023, 1.013, 1.009, 1.005, 1.003, 1.003, 1.001, 1.002, 1.004, 1.006, 1.011, 1.015, 1.022, 1.031, 1.042, 1.056, + 1.024, 1.015, 1.012, 1.008, 1.005, 1.004, 1.004, 1.004, 1.006, 1.009, 1.013, 1.018, 1.024, 1.034, 1.045, 1.057, + 1.027, 1.017, 1.015, 1.012, 1.009, 1.007, 1.007, 1.008, 1.009, 1.013, 1.018, 1.023, 1.029, 1.038, 1.051, 1.061, + 1.029, 1.023, 1.017, 1.015, 1.014, 1.012, 1.011, 1.011, 1.014, 1.018, 1.024, 1.029, 1.036, 1.044, 1.056, 1.066, + 1.034, 1.028, 1.023, 1.022, 1.019, 1.019, 1.018, 1.018, 1.021, 1.025, 1.031, 1.035, 1.042, 1.053, 1.066, 1.074, + 1.041, 1.034, 1.027, 1.025, 1.025, 1.023, 1.023, 1.023, 1.025, 1.031, 1.035, 1.041, 1.049, 1.059, 1.074, 1.079 + ] + } + ], + "calibrations_Cb": [ + { + "ct": 3000, + "table": + [ + 1.001, 1.001, 1.007, 1.015, 1.027, 1.034, 1.038, 1.041, 1.042, 1.043, 1.043, 1.043, 1.041, 1.039, 1.049, 1.054, + 1.011, 1.011, 1.013, 1.023, 1.032, 1.039, 1.044, 1.047, 1.052, 1.056, 1.059, 1.059, 1.055, 1.051, 1.054, 1.056, + 1.015, 1.015, 1.019, 1.032, 1.039, 1.044, 1.047, 1.052, 1.055, 1.059, 1.061, 1.066, 1.063, 1.058, 1.061, 1.064, + 1.016, 1.017, 1.023, 1.032, 1.041, 1.045, 1.048, 1.053, 1.056, 1.061, 1.066, 1.069, 1.067, 1.064, 1.065, 1.068, + 1.018, 1.019, 1.025, 1.033, 1.042, 1.045, 1.049, 1.054, 1.058, 1.063, 1.071, 1.072, 1.071, 1.068, 1.069, 1.071, + 1.023, 1.024, 1.029, 1.035, 1.043, 1.048, 1.052, 1.057, 1.061, 1.065, 1.074, 1.075, 1.075, 1.072, 1.072, 1.075, + 1.027, 1.028, 1.031, 1.038, 1.045, 1.051, 1.054, 1.059, 1.064, 1.068, 1.075, 1.079, 1.078, 1.075, 1.076, 1.081, + 1.029, 1.031, 1.033, 1.044, 1.048, 1.054, 1.059, 1.064, 1.067, 1.073, 1.079, 1.082, 1.082, 1.079, 1.081, 1.085, + 1.033, 1.033, 1.035, 1.047, 1.053, 1.058, 1.064, 1.067, 1.073, 1.079, 1.084, 1.086, 1.086, 1.084, 1.089, 1.091, + 1.037, 1.037, 1.038, 1.049, 1.057, 1.062, 1.068, 1.073, 1.079, 1.084, 1.089, 1.092, 1.092, 1.092, 1.096, 1.104, + 1.041, 1.041, 1.043, 1.051, 1.061, 1.068, 1.073, 1.079, 1.083, 1.089, 1.092, 1.094, 1.097, 1.099, 1.105, 1.115, + 1.048, 1.044, 1.044, 1.051, 1.063, 1.071, 1.076, 1.082, 1.088, 1.091, 1.094, 1.097, 1.099, 1.104, 1.115, 1.126 + ] + }, + { + "ct": 5000, + "table": + [ + 1.001, 1.001, 1.005, 1.011, 1.014, 1.018, 1.019, 1.019, 1.019, 1.021, 1.021, 1.021, 1.019, 1.017, 1.014, 1.014, + 1.009, 1.009, 1.011, 1.014, 1.019, 1.024, 1.026, 1.029, 1.031, 1.032, 1.032, 1.031, 1.027, 1.023, 1.022, 1.022, + 1.011, 1.012, 1.015, 1.018, 1.024, 1.026, 1.029, 1.032, 1.035, 1.036, 1.036, 1.034, 1.031, 1.027, 1.025, 1.025, + 1.012, 1.013, 1.015, 1.019, 1.025, 1.029, 1.032, 1.035, 1.036, 1.038, 1.038, 1.036, 1.034, 1.029, 1.026, 1.026, + 1.013, 1.014, 1.016, 1.019, 1.027, 1.031, 1.034, 1.037, 1.039, 1.039, 1.041, 1.039, 1.036, 1.031, 1.028, 1.027, + 1.014, 1.014, 1.017, 1.021, 1.027, 1.033, 1.037, 1.039, 1.041, 1.041, 1.042, 1.042, 1.039, 1.033, 1.029, 1.028, + 1.015, 1.015, 1.018, 1.021, 1.027, 1.033, 1.037, 1.041, 1.041, 1.042, 1.042, 1.042, 1.039, 1.034, 1.029, 1.029, + 1.015, 1.016, 1.018, 1.022, 1.027, 1.033, 1.037, 1.041, 1.041, 1.042, 1.043, 1.043, 1.041, 1.035, 1.031, 1.031, + 1.015, 1.016, 1.018, 1.022, 1.027, 1.032, 1.037, 1.041, 1.042, 1.042, 1.044, 1.043, 1.041, 1.036, 1.034, 1.033, + 1.016, 1.017, 1.017, 1.022, 1.027, 1.032, 1.036, 1.039, 1.042, 1.042, 1.043, 1.043, 1.041, 1.039, 1.036, 1.034, + 1.017, 1.017, 1.018, 1.022, 1.027, 1.031, 1.035, 1.039, 1.041, 1.042, 1.042, 1.042, 1.042, 1.039, 1.039, 1.039, + 1.018, 1.017, 1.017, 1.021, 1.027, 1.031, 1.033, 1.038, 1.041, 1.041, 1.042, 1.042, 1.041, 1.041, 1.041, 1.041 + ] + } + ], + "luminance_lut": + [ + 2.102, 1.903, 1.658, 1.483, 1.358, 1.267, 1.202, 1.202, 1.202, 1.242, 1.323, 1.431, 1.585, 1.797, 2.096, 2.351, + 1.996, 1.776, 1.549, 1.385, 1.273, 1.204, 1.138, 1.133, 1.133, 1.185, 1.252, 1.343, 1.484, 1.679, 1.954, 2.228, + 1.923, 1.689, 1.474, 1.318, 1.204, 1.138, 1.079, 1.071, 1.071, 1.133, 1.185, 1.284, 1.415, 1.597, 1.854, 2.146, + 1.881, 1.631, 1.423, 1.272, 1.159, 1.079, 1.051, 1.026, 1.046, 1.071, 1.144, 1.245, 1.369, 1.543, 1.801, 2.095, + 1.867, 1.595, 1.391, 1.242, 1.131, 1.051, 1.013, 1.002, 1.013, 1.046, 1.121, 1.219, 1.343, 1.511, 1.752, 2.079, + 1.867, 1.589, 1.385, 1.236, 1.125, 1.048, 1.001, 1.001, 1.003, 1.045, 1.118, 1.217, 1.342, 1.511, 1.746, 2.079, + 1.867, 1.589, 1.385, 1.236, 1.125, 1.048, 1.011, 1.003, 1.011, 1.046, 1.118, 1.217, 1.343, 1.511, 1.746, 2.079, + 1.884, 1.621, 1.411, 1.261, 1.149, 1.071, 1.048, 1.024, 1.046, 1.069, 1.141, 1.239, 1.369, 1.541, 1.781, 2.093, + 1.913, 1.675, 1.459, 1.304, 1.191, 1.125, 1.071, 1.065, 1.069, 1.124, 1.181, 1.278, 1.413, 1.592, 1.842, 2.133, + 1.981, 1.755, 1.529, 1.368, 1.251, 1.191, 1.125, 1.124, 1.124, 1.181, 1.242, 1.337, 1.479, 1.669, 1.935, 2.207, + 2.078, 1.867, 1.625, 1.453, 1.344, 1.251, 1.202, 1.201, 1.201, 1.242, 1.333, 1.418, 1.571, 1.776, 2.063, 2.321, + 2.217, 2.011, 1.747, 1.562, 1.431, 1.331, 1.278, 1.278, 1.278, 1.313, 1.407, 1.523, 1.686, 1.911, 2.226, 2.484 + ], + "sigma": 0.00135, + "sigma_Cb": 0.00279 + } + }, + { + "rpi.contrast": + { + "ce_enable": 1, + "gamma_curve": + [ + 0, 0, + 1024, 5040, + 2048, 9338, + 3072, 12356, + 4096, 15312, + 5120, 18051, + 6144, 20790, + 7168, 23193, + 8192, 25744, + 9216, 27942, + 10240, 30035, + 11264, 32005, + 12288, 33975, + 13312, 35815, + 14336, 37600, + 15360, 39168, + 16384, 40642, + 18432, 43379, + 20480, 45749, + 22528, 47753, + 24576, 49621, + 26624, 51253, + 28672, 52698, + 30720, 53796, + 32768, 54876, + 36864, 57012, + 40960, 58656, + 45056, 59954, + 49152, 61183, + 53248, 62355, + 57344, 63419, + 61440, 64476, + 65535, 65535 + ] + } + }, + { + "rpi.ccm": + { + "ccms": [ + { + "ct": 2698, + "ccm": + [ + 1.57227, -0.32596, -0.24631, + -0.61264, 1.70791, -0.09526, + -0.43254, 0.48489, 0.94765 + ] + }, + { + "ct": 2930, + "ccm": + [ + 1.69455, -0.52724, -0.16731, + -0.67131, 1.78468, -0.11338, + -0.41609, 0.54693, 0.86916 + ] + }, + { + "ct": 3643, + "ccm": + [ + 1.74041, -0.77553, 0.03512, + -0.44073, 1.34131, 0.09943, + -0.11035, -0.93919, 2.04954 + ] + }, + { + "ct": 4605, + "ccm": + [ + 1.49865, -0.41638, -0.08227, + -0.39445, 1.70114, -0.30669, + 0.01319, -0.88009, 1.86689 + ] + }, + { + "ct": 5658, + "ccm": + [ + 1.38601, -0.23128, -0.15472, + -0.37641, 1.70444, -0.32803, + -0.01575, -0.71466, 1.73041 + ] + } + ] + } + }, + { + "rpi.sharpen": { } + } + ] +} diff --git a/src/ipa/rpi/vc4/data/meson.build b/src/ipa/rpi/vc4/data/meson.build index 94c0ee6e..7a8001ee 100644 --- a/src/ipa/rpi/vc4/data/meson.build +++ b/src/ipa/rpi/vc4/data/meson.build @@ -9,6 +9,7 @@ conf_files = files([ 'imx296_mono.json', 'imx327.json', 'imx378.json', + 'imx415.json', 'imx462.json', 'imx477.json', 'imx477_noir.json', |