diff options
25 files changed, 1377 insertions, 106 deletions
diff --git a/src/ipa/libipa/awb.cpp b/src/ipa/libipa/awb.cpp new file mode 100644 index 00000000..6157bd43 --- /dev/null +++ b/src/ipa/libipa/awb.cpp @@ -0,0 +1,265 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Ideas on Board Oy + * + * Generic AWB algorithms + */ + +#include "awb.h" + +#include <libcamera/base/log.h> + +#include <libcamera/control_ids.h> + +/** + * \file awb.h + * \brief Base classes for AWB algorithms + */ + +namespace libcamera { + +LOG_DEFINE_CATEGORY(Awb) + +namespace ipa { + +/** + * \class AwbResult + * \brief The result of an awb calculation + * + * This class holds the result of an auto white balance calculation. + */ + +/** + * \var AwbResult::gains + * \brief The calculated white balance gains + */ + +/** + * \var AwbResult::colourTemperature + * \brief The calculated colour temperature in Kelvin + */ + +/** + * \class AwbStats + * \brief An abstraction class wrapping hardware-specific AWB statistics + * + * Pipeline handlers using an AWB algorithm based on the AwbAlgorithm class need + * to implement this class to give the algorithm access to the hardware-specific + * statistics data. + */ + +/** + * \fn AwbStats::computeColourError + * \brief Compute an error value for when the given gains would be applied + * \param[in] gains The gains to apply + * + * Compute an error value (non-greyness) assuming the given \a gains would be + * applied. To keep the actual implementations computationally inexpensive, + * the squared colour error shall be returned. + * + * If the awb statistics provide multiple zones, the average of the individual + * squared errors shall be returned. Averaging/normalizing is necessary so that + * the numeric dimensions are the same on all hardware platforms. + * + * \return The computed error value + */ + +/** + * \fn AwbStats::getRGBMeans + * \brief Get RGB means of the statistics + * + * Fetch the RGB means from the statistics. The values of each channel are + * dimensionless and only the ratios are used for further calculations. This is + * used by the simple gray world model to calculate the gains to apply. + * + * \return The RGB means + */ + +/** + * \class AwbAlgorithm + * \brief A base class for auto white balance algorithms + * + * This class is a base class for auto white balance algorithms. It defines an + * interface for the algorithms to implement, and is used by the IPAs to + * interact with the concrete implementation. + */ + +/** + * \fn AwbAlgorithm::init + * \brief Initialize the algorithm with the given tuning data + * \param[in] tuningData The tuning data to use for the algorithm + * + * \return 0 on success, a negative error code otherwise + */ + +/** + * \fn AwbAlgorithm::calculateAwb + * \brief Calculate awb data from the given statistics + * \param[in] stats The statistics to use for the calculation + * \param[in] lux The lux value of the scene + * + * Calculate an AwbResult object from the given statistics and lux value. A \a + * lux value of 0 means it is unknown or invalid and the algorithm shall ignore + * it. + * + * \return The awb result + */ + +/** + * \fn AwbAlgorithm::gainsFromColourTemperature + * \brief Compute white balance gains from a colour temperature + * \param[in] colourTemperature The colour temperature in Kelvin + * + * Compute the white balance gains from a \a colourTemperature. This function + * does not take any statistics into account. It is used to compute the colour + * gains when the user manually specifies a colour temperature. + * + * \return The colour gains + */ + +/** + * \fn AwbAlgorithm::controls + * \brief Get the controls info map for this algorithm + * + * \return The controls info map + */ + +/** + * \fn AwbAlgorithm::handleControls + * \param[in] controls The controls to handle + * \brief Handle the controls supplied in a request + */ + +/** + * \var AwbAlgorithm::controls_ + * \brief Controls info map for the controls provided by the algorithm + */ + +/** + * \var AwbAlgorithm::modes_ + * \brief Map of all configured modes + * \sa AwbAlgorithm::parseModeConfigs + */ + +/** + * \class AwbAlgorithm::ModeConfig + * \brief Holds the configuration of a single AWB mode + * + * Awb modes limit the regulation of the AWB algorithm to a specific range of + * colour temperatures. + */ + +/** + * \var AwbAlgorithm::ModeConfig::ctLo + * \brief The lowest valid colour temperature of that mode + */ + +/** + * \var AwbAlgorithm::ModeConfig::ctHi + * \brief The highest valid colour temperature of that mode + */ + +/** + * \brief Parse the mode configurations from the tuning data + * \param[in] tuningData the YamlObject representing the tuning data + * \param[in] def The default value for the AwbMode control + * + * Utility function to parse the tuning data for an AwbMode entry and read all + * provided modes. It adds controls::AwbMode to AwbAlgorithm::controls_ and + * populates AwbAlgorithm::modes_. For a list of possible modes see \ref + * controls::AwbModeEnum. + * + * Each mode entry must contain a "lo" and "hi" key to specify the lower and + * upper colour temperature of that mode. For example: + * + * \code{.unparsed} + * algorithms: + * - Awb: + * AwbMode: + * AwbAuto: + * lo: 2500 + * hi: 8000 + * AwbIncandescent: + * lo: 2500 + * hi: 3000 + * ... + * \endcode + * + * If \a def is supplied but not contained in the the \a tuningData, -EINVAL is + * returned. + * + * \sa controls::AwbModeEnum + * \return Zero on success, negative error code otherwise + */ +int AwbAlgorithm::parseModeConfigs(const YamlObject &tuningData, + const ControlValue &def) +{ + std::vector<ControlValue> availableModes; + + const YamlObject &yamlModes = tuningData[controls::AwbMode.name()]; + if (!yamlModes.isDictionary()) { + LOG(Awb, Error) + << "AwbModes must be a dictionary."; + return -EINVAL; + } + + for (const auto &[modeName, modeDict] : yamlModes.asDict()) { + if (controls::AwbModeNameValueMap.find(modeName) == + controls::AwbModeNameValueMap.end()) { + LOG(Awb, Warning) + << "Skipping unknown awb mode '" + << modeName << "'"; + continue; + } + + if (!modeDict.isDictionary()) { + LOG(Awb, Error) + << "Invalid awb mode '" << modeName << "'"; + return -EINVAL; + } + + const auto &modeValue = static_cast<controls::AwbModeEnum>( + controls::AwbModeNameValueMap.at(modeName)); + + ModeConfig &config = modes_[modeValue]; + + auto hi = modeDict["hi"].get<double>(); + if (!hi) { + LOG(Awb, Error) << "Failed to read hi param of mode " + << modeName; + return -EINVAL; + } + config.ctHi = *hi; + + auto lo = modeDict["lo"].get<double>(); + if (!lo) { + LOG(Awb, Error) << "Failed to read low param of mode " + << modeName; + return -EINVAL; + } + config.ctLo = *lo; + + availableModes.push_back(modeValue); + } + + if (modes_.empty()) { + LOG(Awb, Error) << "No AWB modes configured"; + return -EINVAL; + } + + if (!def.isNone() && + modes_.find(def.get<controls::AwbModeEnum>()) == modes_.end()) { + const auto &names = controls::AwbMode.enumerators(); + LOG(Awb, Error) << names.at(def.get<controls::AwbModeEnum>()) + << " mode is missing in the configuration."; + return -EINVAL; + } + + controls_[&controls::AwbMode] = ControlInfo(availableModes, def); + + return 0; +} + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/awb.h b/src/ipa/libipa/awb.h new file mode 100644 index 00000000..4a1b012a --- /dev/null +++ b/src/ipa/libipa/awb.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Ideas on Board Oy + * + * Generic AWB algorithms + */ + +#pragma once + +#include <map> + +#include <libcamera/control_ids.h> +#include <libcamera/controls.h> + +#include "libcamera/internal/vector.h" +#include "libcamera/internal/yaml_parser.h" + +namespace libcamera { + +namespace ipa { + +struct AwbResult { + RGB<double> gains; + double colourTemperature; +}; + +struct AwbStats { + virtual double computeColourError(const RGB<double> &gains) const = 0; + virtual RGB<double> getRGBMeans() const = 0; +}; + +class AwbAlgorithm +{ +public: + virtual ~AwbAlgorithm() = default; + + virtual int init(const YamlObject &tuningData) = 0; + virtual AwbResult calculateAwb(const AwbStats &stats, int lux) = 0; + virtual RGB<double> gainsFromColourTemperature(double colourTemperature) = 0; + + const ControlInfoMap::Map &controls() const + { + return controls_; + } + + virtual void handleControls([[maybe_unused]] const ControlList &controls) {} + +protected: + int parseModeConfigs(const YamlObject &tuningData, + const ControlValue &def = {}); + + struct ModeConfig { + double ctHi; + double ctLo; + }; + + ControlInfoMap::Map controls_; + std::map<controls::AwbModeEnum, AwbAlgorithm::ModeConfig> modes_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/awb_bayes.cpp b/src/ipa/libipa/awb_bayes.cpp new file mode 100644 index 00000000..e75bfcd6 --- /dev/null +++ b/src/ipa/libipa/awb_bayes.cpp @@ -0,0 +1,499 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2019, Raspberry Pi Ltd + * Copyright (C) 2024 Ideas on Board Oy + * + * Implementation of a bayesian AWB algorithm + */ + +#include "awb_bayes.h" + +#include <cmath> + +#include <libcamera/base/log.h> +#include <libcamera/control_ids.h> + +#include "colours.h" + +/** + * \file awb_bayes.h + * \brief Implementation of bayesian auto white balance algorithm + * + * This implementation is based on the initial implementation done by + * RaspberryPi. + * \todo: Documentation + * + * \todo Not all the features implemented by RaspberryPi were ported over to + * this algorithm because they either rely on hardware features not generally + * available or were considered not important enough at the moment. + * + * The following parts are not implemented: + * + * - min_pixels: minimum proportion of pixels counted within AWB region for it + * to be "useful" + * - min_g: minimum G value of those pixels, to be regarded a "useful" + * - min_regions: number of AWB regions that must be "useful" in order to do the + * AWB calculation + * - deltaLimit: clamp on colour error term (so as not to penalize non-grey + * excessively) + * - bias_proportion: The biasProportion parameter adds a small proportion of + * the counted pixels to a region biased to the biasCT colour temperature. + * A typical value for biasProportion would be between 0.05 to 0.1. + * - bias_ct: CT target for the search bias + * - sensitivityR: red sensitivity ratio (set to canonical sensor's R/G divided + * by this sensor's R/G) + * - sensitivityB: blue sensitivity ratio (set to canonical sensor's B/G divided + * by this sensor's B/G) + */ + +namespace libcamera { + +LOG_DECLARE_CATEGORY(Awb) + +namespace { + +template<typename T> +class LimitsRecorder +{ +public: + LimitsRecorder() + : min_(std::numeric_limits<T>::max()), + max_(std::numeric_limits<T>::min()) + { + } + + void record(const T &value) + { + min_ = std::min(min_, value); + max_ = std::max(max_, value); + } + + const T &min() const { return min_; } + const T &max() const { return max_; } + +private: + T min_; + T max_; +}; + +#ifndef __DOXYGEN__ +template<typename T> +std::ostream &operator<<(std::ostream &out, const LimitsRecorder<T> &v) +{ + out << "[ " << v.min() << ", " << v.max() << " ]"; + return out; +} +#endif + +} /* namespace */ + +namespace ipa { + +/** + * \brief Step size control for CT search + */ +constexpr double kSearchStep = 0.2; + +/** + * \copydoc libcamera::ipa::Interpolator::interpolate() + */ +template<> +void Interpolator<Pwl>::interpolate(const Pwl &a, const Pwl &b, Pwl &dest, double lambda) +{ + dest = Pwl::combine(a, b, + [=](double /*x*/, double y0, double y1) -> double { + return y0 * (1.0 - lambda) + y1 * lambda; + }); +} + +/** + * \class AwbBayes + * \brief Implementation of a bayesian auto white balance algorithm + * + * In a bayesian AWB algorithm the auto white balance estimation is improved by + * taking the likelihood of a given lightsource based on the estimated lux level + * into account. E.g. If it is very bright we can assume that we are outside and + * that colour temperatures around 6500 are preferred. + * + * The second part of this algorithm is the search for the most likely colour + * temperature. It is implemented in AwbBayes::coarseSearch() and in + * AwbBayes::fineSearch(). The search works very well without prior likelihoods + * and therefore the algorithm itself provides very good results even without + * prior likelihoods. + */ + +/** + * \var AwbBayes::transversePos_ + * \brief How far to wander off CT curve towards "more purple" + */ + +/** + * \var AwbBayes::transverseNeg_ + * \brief How far to wander off CT curve towards "more green" + */ + +/** + * \var AwbBayes::currentMode_ + * \brief The currently selected mode + */ + +int AwbBayes::init(const YamlObject &tuningData) +{ + int ret = colourGainCurve_.readYaml(tuningData["colourGains"], "ct", "gains"); + if (ret) { + LOG(Awb, Error) + << "Failed to parse 'colourGains' " + << "parameter from tuning file"; + return ret; + } + + ctR_.clear(); + ctB_.clear(); + for (const auto &[ct, g] : colourGainCurve_.data()) { + ctR_.append(ct, 1.0 / g[0]); + ctB_.append(ct, 1.0 / g[1]); + } + + /* We will want the inverse functions of these too. */ + ctRInverse_ = ctR_.inverse().first; + ctBInverse_ = ctB_.inverse().first; + + ret = readPriors(tuningData); + if (ret) { + LOG(Awb, Error) << "Failed to read priors"; + return ret; + } + + ret = parseModeConfigs(tuningData, controls::AwbAuto); + if (ret) { + LOG(Awb, Error) + << "Failed to parse mode parameter from tuning file"; + return ret; + } + currentMode_ = &modes_[controls::AwbAuto]; + + transversePos_ = tuningData["transversePos"].get<double>(0.01); + transverseNeg_ = tuningData["transverseNeg"].get<double>(0.01); + if (transversePos_ <= 0 || transverseNeg_ <= 0) { + LOG(Awb, Error) << "AwbConfig: transversePos/Neg must be > 0"; + return -EINVAL; + } + + return 0; +} + +int AwbBayes::readPriors(const YamlObject &tuningData) +{ + const auto &priorsList = tuningData["priors"]; + std::map<uint32_t, Pwl> priors; + + if (!priorsList) { + LOG(Awb, Error) << "No priors specified"; + return -EINVAL; + } + + for (const auto &p : priorsList.asList()) { + if (!p.contains("lux")) { + LOG(Awb, Error) << "Missing lux value"; + return -EINVAL; + } + + uint32_t lux = p["lux"].get<uint32_t>(0); + if (priors.count(lux)) { + LOG(Awb, Error) << "Duplicate prior for lux value " << lux; + return -EINVAL; + } + + std::vector<uint32_t> temperatures = + p["ct"].getList<uint32_t>().value_or(std::vector<uint32_t>{}); + std::vector<double> probabilities = + p["probability"].getList<double>().value_or(std::vector<double>{}); + + if (temperatures.size() != probabilities.size()) { + LOG(Awb, Error) + << "Ct and probability array sizes are unequal"; + return -EINVAL; + } + + if (temperatures.empty()) { + LOG(Awb, Error) + << "Ct and probability arrays are empty"; + return -EINVAL; + } + + std::map<int, double> ctToProbability; + for (unsigned int i = 0; i < temperatures.size(); i++) { + int t = temperatures[i]; + if (ctToProbability.count(t)) { + LOG(Awb, Error) << "Duplicate ct value"; + return -EINVAL; + } + + ctToProbability[t] = probabilities[i]; + } + + auto &pwl = priors[lux]; + for (const auto &[ct, prob] : ctToProbability) { + if (prob < 1e-6) { + LOG(Awb, Error) << "Prior probability must be larger than 1e-6"; + return -EINVAL; + } + pwl.append(ct, prob); + } + } + + if (priors.empty()) { + LOG(Awb, Error) << "No priors specified"; + return -EINVAL; + } + + priors_.setData(std::move(priors)); + + return 0; +} + +void AwbBayes::handleControls(const ControlList &controls) +{ + auto mode = controls.get(controls::AwbMode); + if (mode) { + auto it = modes_.find(static_cast<controls::AwbModeEnum>(*mode)); + if (it != modes_.end()) + currentMode_ = &it->second; + else + LOG(Awb, Error) << "Unsupported AWB mode " << *mode; + } +} + +RGB<double> AwbBayes::gainsFromColourTemperature(double colourTemperature) +{ + /* + * \todo: In the RaspberryPi code, the ct curve was interpolated in + * the white point space (1/x) not in gains space. This feels counter + * intuitive, as the gains are in linear space. But I can't prove it. + */ + const auto &gains = colourGainCurve_.getInterpolated(colourTemperature); + return { { gains[0], 1.0, gains[1] } }; +} + +AwbResult AwbBayes::calculateAwb(const AwbStats &stats, int lux) +{ + ipa::Pwl prior; + if (lux > 0) { + prior = priors_.getInterpolated(lux); + prior.map([](double x, double y) { + LOG(Awb, Debug) << "(" << x << "," << y << ")"; + }); + } else { + prior.append(0, 1.0); + } + + double t = coarseSearch(prior, stats); + double r = ctR_.eval(t); + double b = ctB_.eval(t); + LOG(Awb, Debug) + << "After coarse search: r " << r << " b " << b << " (gains r " + << 1 / r << " b " << 1 / b << ")"; + + /* + * Original comment from RaspberryPi: + * Not entirely sure how to handle the fine search yet. Mostly the + * estimated CT is already good enough, but the fine search allows us to + * wander transversely off the CT curve. Under some illuminants, where + * there may be more or less green light, this may prove beneficial, + * though I probably need more real datasets before deciding exactly how + * this should be controlled and tuned. + */ + fineSearch(t, r, b, prior, stats); + LOG(Awb, Debug) + << "After fine search: r " << r << " b " << b << " (gains r " + << 1 / r << " b " << 1 / b << ")"; + + return { { { 1.0 / r, 1.0, 1.0 / b } }, t }; +} + +double AwbBayes::coarseSearch(const ipa::Pwl &prior, const AwbStats &stats) const +{ + std::vector<Pwl::Point> points; + size_t bestPoint = 0; + double t = currentMode_->ctLo; + int spanR = -1; + int spanB = -1; + LimitsRecorder<double> errorLimits; + LimitsRecorder<double> priorLogLikelihoodLimits; + + /* Step down the CT curve evaluating log likelihood. */ + while (true) { + double r = ctR_.eval(t, &spanR); + double b = ctB_.eval(t, &spanB); + RGB<double> gains({ 1 / r, 1.0, 1 / b }); + double delta2Sum = stats.computeColourError(gains); + double priorLogLikelihood = log(prior.eval(prior.domain().clamp(t))); + double finalLogLikelihood = delta2Sum - priorLogLikelihood; + + errorLimits.record(delta2Sum); + priorLogLikelihoodLimits.record(priorLogLikelihood); + + points.push_back({ { t, finalLogLikelihood } }); + if (points.back().y() < points[bestPoint].y()) + bestPoint = points.size() - 1; + + if (t == currentMode_->ctHi) + break; + + /* + * Ensure even steps along the r/b curve by scaling them by the + * current t. + */ + t = std::min(t + t / 10 * kSearchStep, currentMode_->ctHi); + } + + t = points[bestPoint].x(); + LOG(Awb, Debug) << "Coarse search found CT " << t + << " error limits:" << errorLimits + << " prior log likelihood limits:" << priorLogLikelihoodLimits; + + /* + * We have the best point of the search, but refine it with a quadratic + * interpolation around its neighbors. + */ + if (points.size() > 2) { + bestPoint = std::clamp(bestPoint, std::size_t{ 1 }, points.size() - 2); + t = interpolateQuadratic(points[bestPoint - 1], + points[bestPoint], + points[bestPoint + 1]); + LOG(Awb, Debug) + << "After quadratic refinement, coarse search has CT " + << t; + } + + return t; +} + +void AwbBayes::fineSearch(double &t, double &r, double &b, ipa::Pwl const &prior, const AwbStats &stats) const +{ + int spanR = -1; + int spanB = -1; + double step = t / 10 * kSearchStep * 0.1; + int nsteps = 5; + ctR_.eval(t, &spanR); + ctB_.eval(t, &spanB); + double rDiff = ctR_.eval(t + nsteps * step, &spanR) - + ctR_.eval(t - nsteps * step, &spanR); + double bDiff = ctB_.eval(t + nsteps * step, &spanB) - + ctB_.eval(t - nsteps * step, &spanB); + Pwl::Point transverse({ bDiff, -rDiff }); + if (transverse.length2() < 1e-6) + return; + /* + * transverse is a unit vector orthogonal to the b vs. r function + * (pointing outwards with r and b increasing) + */ + transverse = transverse / transverse.length(); + double bestLogLikelihood = 0; + double bestT = 0; + Pwl::Point bestRB(0); + double transverseRange = transverseNeg_ + transversePos_; + const int maxNumDeltas = 12; + LimitsRecorder<double> errorLimits; + LimitsRecorder<double> priorLogLikelihoodLimits; + + + /* a transverse step approximately every 0.01 r/b units */ + int numDeltas = floor(transverseRange * 100 + 0.5) + 1; + numDeltas = std::clamp(numDeltas, 3, maxNumDeltas); + + /* + * Step down CT curve. March a bit further if the transverse range is + * large. + */ + nsteps += numDeltas; + for (int i = -nsteps; i <= nsteps; i++) { + double tTest = t + i * step; + double priorLogLikelihood = + log(prior.eval(prior.domain().clamp(tTest))); + priorLogLikelihoodLimits.record(priorLogLikelihood); + Pwl::Point rbStart{ { ctR_.eval(tTest, &spanR), + ctB_.eval(tTest, &spanB) } }; + Pwl::Point samples[maxNumDeltas]; + int bestPoint = 0; + + /* + * Sample numDeltas points transversely *off* the CT curve + * in the range [-transverseNeg, transversePos]. + * The x values of a sample contains the distance and the y + * value contains the corresponding log likelihood. + */ + double transverseStep = transverseRange / (numDeltas - 1); + for (int j = 0; j < numDeltas; j++) { + auto &p = samples[j]; + p.x() = -transverseNeg_ + transverseStep * j; + Pwl::Point rbTest = rbStart + transverse * p.x(); + RGB<double> gains({ 1 / rbTest[0], 1.0, 1 / rbTest[1] }); + double delta2Sum = stats.computeColourError(gains); + errorLimits.record(delta2Sum); + p.y() = delta2Sum - priorLogLikelihood; + + if (p.y() < samples[bestPoint].y()) + bestPoint = j; + } + + /* + * We have all samples transversely across the CT curve, + * now let's do a quadratic interpolation for the best result. + */ + bestPoint = std::clamp(bestPoint, 1, numDeltas - 2); + double bestOffset = interpolateQuadratic(samples[bestPoint - 1], + samples[bestPoint], + samples[bestPoint + 1]); + Pwl::Point rbTest = rbStart + transverse * bestOffset; + RGB<double> gains({ 1 / rbTest[0], 1.0, 1 / rbTest[1] }); + double delta2Sum = stats.computeColourError(gains); + errorLimits.record(delta2Sum); + double finalLogLikelihood = delta2Sum - priorLogLikelihood; + + if (bestT == 0 || finalLogLikelihood < bestLogLikelihood) { + bestLogLikelihood = finalLogLikelihood; + bestT = tTest; + bestRB = rbTest; + } + } + + t = bestT; + r = bestRB[0]; + b = bestRB[1]; + LOG(Awb, Debug) + << "Fine search found t " << t << " r " << r << " b " << b + << " error limits: " << errorLimits + << " prior log likelihood limits: " << priorLogLikelihoodLimits; +} + +/** + * \brief Find extremum of function + * \param[in] a First point + * \param[in] b Second point + * \param[in] c Third point + * + * Given 3 points on a curve, find the extremum of the function in that interval + * by fitting a quadratic. + * + * \return The x value of the extremum clamped to the interval [a.x(), c.x()] + */ +double AwbBayes::interpolateQuadratic(Pwl::Point const &a, Pwl::Point const &b, + Pwl::Point const &c) const +{ + const double eps = 1e-3; + Pwl::Point ca = c - a; + Pwl::Point ba = b - a; + double denominator = 2 * (ba.y() * ca.x() - ca.y() * ba.x()); + if (std::abs(denominator) > eps) { + double numerator = ba.y() * ca.x() * ca.x() - ca.y() * ba.x() * ba.x(); + double result = numerator / denominator + a.x(); + return std::max(a.x(), std::min(c.x(), result)); + } + /* has degenerated to straight line segment */ + return a.y() < c.y() - eps ? a.x() : (c.y() < a.y() - eps ? c.x() : b.x()); +} + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/awb_bayes.h b/src/ipa/libipa/awb_bayes.h new file mode 100644 index 00000000..47db7243 --- /dev/null +++ b/src/ipa/libipa/awb_bayes.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Ideas on Board Oy + * + * Base class for bayes AWB algorithms + */ + +#pragma once + +#include <map> +#include <memory> +#include <tuple> +#include <vector> + +#include <libcamera/base/utils.h> + +#include <libcamera/control_ids.h> +#include <libcamera/controls.h> + +#include "libcamera/internal/vector.h" +#include "libcamera/internal/yaml_parser.h" + +#include "awb.h" +#include "interpolator.h" +#include "pwl.h" + +namespace libcamera { + +namespace ipa { + +class AwbBayes : public AwbAlgorithm +{ +public: + AwbBayes() = default; + + int init(const YamlObject &tuningData) override; + AwbResult calculateAwb(const AwbStats &stats, int lux) override; + RGB<double> gainsFromColourTemperature(double temperatureK) override; + void handleControls(const ControlList &controls) override; + +private: + int readPriors(const YamlObject &tuningData); + + void fineSearch(double &t, double &r, double &b, ipa::Pwl const &prior, + const AwbStats &stats) const; + double coarseSearch(const ipa::Pwl &prior, const AwbStats &stats) const; + double interpolateQuadratic(ipa::Pwl::Point const &a, + ipa::Pwl::Point const &b, + ipa::Pwl::Point const &c) const; + + Interpolator<Pwl> priors_; + Interpolator<Vector<double, 2>> colourGainCurve_; + + ipa::Pwl ctR_; + ipa::Pwl ctB_; + ipa::Pwl ctRInverse_; + ipa::Pwl ctBInverse_; + + double transversePos_; + double transverseNeg_; + + ModeConfig *currentMode_ = nullptr; +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/awb_grey.cpp b/src/ipa/libipa/awb_grey.cpp new file mode 100644 index 00000000..49448976 --- /dev/null +++ b/src/ipa/libipa/awb_grey.cpp @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Ideas on Board Oy + * + * Base class for grey world AWB algorithm + */ + +#include "awb_grey.h" + +#include <cmath> + +#include <libcamera/base/log.h> +#include <libcamera/control_ids.h> + +#include "colours.h" + +using namespace libcamera::controls; + +/** + * \file awb_grey.h + * \brief Implementation of a grey world AWB algorithm + */ + +namespace libcamera { + +LOG_DECLARE_CATEGORY(Awb) +namespace ipa { + +/** + * \class AwbGrey + * \brief A Grey world auto white balance algorithm + */ + +/** + * \brief Initialize the algorithm with the given tuning data + * \param[in] tuningData The tuning data for the algorithm + * + * Load the colour temperature curve from the tuning data. If there is no tuning + * data available, continue with a warning. Manual colour temperature will not + * work in that case. + * + * \return 0 on success, a negative error code otherwise + */ +int AwbGrey::init(const YamlObject &tuningData) +{ + Interpolator<Vector<double, 2>> gains; + int ret = gains.readYaml(tuningData["colourGains"], "ct", "gains"); + if (ret < 0) + LOG(Awb, Warning) + << "Failed to parse 'colourGains' " + << "parameter from tuning file; " + << "manual colour temperature will not work properly"; + else + colourGainCurve_ = gains; + + return 0; +} + +/** + * \brief Calculate awb data from the given statistics + * \param[in] stats The statistics to use for the calculation + * \param[in] lux The lux value of the scene + * + * Estimates the colour temperature based on the colours::estimateCCT function. + * The gains are calculated purely based on the RGB means provided by the \a + * stats. The colour temperature is not taken into account when calculating the + * gains. + * + * The \a lux parameter is not used in this algorithm. + * + * \return The awb result + */ +AwbResult AwbGrey::calculateAwb(const AwbStats &stats, [[maybe_unused]] int lux) +{ + AwbResult result; + auto means = stats.getRGBMeans(); + result.colourTemperature = estimateCCT(means); + + /* + * Estimate the red and blue gains to apply in a grey world. The green + * gain is hardcoded to 1.0. Avoid divisions by zero by clamping the + * divisor to a minimum value of 1.0. + */ + result.gains.r() = means.g() / std::max(means.r(), 1.0); + result.gains.g() = 1.0; + result.gains.b() = means.g() / std::max(means.b(), 1.0); + return result; +} + +/** + * \brief Compute white balance gains from a colour temperature + * \param[in] colourTemperature The colour temperature in Kelvin + * + * Compute the white balance gains from a \a colourTemperature. This function + * does not take any statistics into account. It simply interpolates the colour + * gains configured in the colour temperature curve. + * + * \return The colour gains if a colour temperature curve is available, + * [1, 1, 1] otherwise. + */ +RGB<double> AwbGrey::gainsFromColourTemperature(double colourTemperature) +{ + if (!colourGainCurve_) { + LOG(Awb, Error) << "No gains defined"; + return RGB<double>({ 1.0, 1.0, 1.0 }); + } + + auto gains = colourGainCurve_->getInterpolated(colourTemperature); + return { { gains[0], 1.0, gains[1] } }; +} + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/awb_grey.h b/src/ipa/libipa/awb_grey.h new file mode 100644 index 00000000..1a365e61 --- /dev/null +++ b/src/ipa/libipa/awb_grey.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Ideas on Board Oy + * + * AWB grey world algorithm + */ + +#pragma once + +#include "libcamera/internal/vector.h" +#include "libcamera/internal/yaml_parser.h" + +#include "awb.h" +#include "interpolator.h" + +namespace libcamera { + +namespace ipa { + +class AwbGrey : public AwbAlgorithm +{ +public: + AwbGrey() = default; + + int init(const YamlObject &tuningData) override; + AwbResult calculateAwb(const AwbStats &stats, int lux) override; + RGB<double> gainsFromColourTemperature(double colourTemperature) override; + +private: + std::optional<Interpolator<Vector<double, 2>>> colourGainCurve_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/interpolator.cpp b/src/ipa/libipa/interpolator.cpp index 73e8d3b7..f901a86e 100644 --- a/src/ipa/libipa/interpolator.cpp +++ b/src/ipa/libipa/interpolator.cpp @@ -126,6 +126,13 @@ namespace ipa { */ /** + * \fn std::map<unsigned int, T> &Interpolator<T>::data() const + * \brief Access the internal map + * + * \return The internal map + */ + +/** * \fn const T& Interpolator<T>::getInterpolated() * \brief Retrieve an interpolated value for the given key * \param[in] key The unsigned integer key of the object to retrieve @@ -136,8 +143,7 @@ namespace ipa { */ /** - * \fn void Interpolator<T>::interpolate(const T &a, const T &b, T &dest, double - * lambda) + * \fn void Interpolator<T>::interpolate(const T &a, const T &b, T &dest, double lambda) * \brief Interpolate between two instances of T * \param a The first value to interpolate * \param b The second value to interpolate diff --git a/src/ipa/libipa/interpolator.h b/src/ipa/libipa/interpolator.h index fffce214..7880db69 100644 --- a/src/ipa/libipa/interpolator.h +++ b/src/ipa/libipa/interpolator.h @@ -81,6 +81,11 @@ public: lastInterpolatedKey_.reset(); } + const std::map<unsigned int, T> &data() const + { + return data_; + } + const T &getInterpolated(unsigned int key, unsigned int *quantizedKey = nullptr) { ASSERT(data_.size() > 0); diff --git a/src/ipa/libipa/lux.cpp b/src/ipa/libipa/lux.cpp index 61f8fea8..899e8824 100644 --- a/src/ipa/libipa/lux.cpp +++ b/src/ipa/libipa/lux.cpp @@ -44,11 +44,6 @@ namespace ipa { */ /** - * \var Lux::binSize_ - * \brief The maximum count of each bin - */ - -/** * \var Lux::referenceExposureTime_ * \brief The exposure time of the reference image, in microseconds */ @@ -65,9 +60,8 @@ namespace ipa { /** * \var Lux::referenceY_ - * \brief The measured luminance of the reference image, out of the bin size + * \brief The measured luminance of the reference image, normalized to 1 * - * \sa binSize_ */ /** @@ -76,11 +70,9 @@ namespace ipa { */ /** - * \brief Construct the Lux helper module - * \param[in] binSize The maximum count of each bin - */ -Lux::Lux(unsigned int binSize) - : binSize_(binSize) + * \brief Construct the Lux helper module + */ +Lux::Lux() { } @@ -97,7 +89,7 @@ Lux::Lux(unsigned int binSize) * referenceExposureTime: 10000 * referenceAnalogueGain: 4.0 * referenceDigitalGain: 1.0 - * referenceY: 12000 + * referenceY: 0.1831 * referenceLux: 1000 * \endcode * @@ -167,7 +159,7 @@ double Lux::estimateLux(utils::Duration exposureTime, double exposureTimeRatio = referenceExposureTime_ / exposureTime; double aGainRatio = referenceAnalogueGain_ / aGain; double dGainRatio = referenceDigitalGain_ / dGain; - double yRatio = currentY * (binSize_ / yHist.bins()) / referenceY_; + double yRatio = (currentY / yHist.bins()) / referenceY_; double estimatedLux = exposureTimeRatio * aGainRatio * dGainRatio * yRatio * referenceLux_; diff --git a/src/ipa/libipa/lux.h b/src/ipa/libipa/lux.h index 93ca6479..d95bcdaf 100644 --- a/src/ipa/libipa/lux.h +++ b/src/ipa/libipa/lux.h @@ -21,7 +21,7 @@ class Histogram; class Lux { public: - Lux(unsigned int binSize); + Lux(); int parseTuningData(const YamlObject &tuningData); double estimateLux(utils::Duration exposureTime, @@ -29,7 +29,6 @@ public: const Histogram &yHist) const; private: - unsigned int binSize_; utils::Duration referenceExposureTime_; double referenceAnalogueGain_; double referenceDigitalGain_; diff --git a/src/ipa/libipa/meson.build b/src/ipa/libipa/meson.build index 12d8d15b..660be940 100644 --- a/src/ipa/libipa/meson.build +++ b/src/ipa/libipa/meson.build @@ -3,6 +3,9 @@ libipa_headers = files([ 'agc_mean_luminance.h', 'algorithm.h', + 'awb_bayes.h', + 'awb_grey.h', + 'awb.h', 'camera_sensor_helper.h', 'colours.h', 'exposure_mode_helper.h', @@ -19,6 +22,9 @@ libipa_headers = files([ libipa_sources = files([ 'agc_mean_luminance.cpp', 'algorithm.cpp', + 'awb_bayes.cpp', + 'awb_grey.cpp', + 'awb.cpp', 'camera_sensor_helper.cpp', 'colours.cpp', 'exposure_mode_helper.cpp', diff --git a/src/ipa/libipa/pwl.cpp b/src/ipa/libipa/pwl.cpp index 88fe2022..3fa005ba 100644 --- a/src/ipa/libipa/pwl.cpp +++ b/src/ipa/libipa/pwl.cpp @@ -160,6 +160,11 @@ void Pwl::prepend(double x, double y, const double eps) */ /** + * \fn Pwl::clear() + * \brief Clear the piecewise linear function + */ + +/** * \fn Pwl::size() const * \brief Retrieve the number of points in the piecewise linear function * \return The number of points in the piecewise linear function diff --git a/src/ipa/libipa/pwl.h b/src/ipa/libipa/pwl.h index 8fdc7053..c1496c30 100644 --- a/src/ipa/libipa/pwl.h +++ b/src/ipa/libipa/pwl.h @@ -49,6 +49,7 @@ public: void append(double x, double y, double eps = 1e-6); bool empty() const { return points_.empty(); } + void clear() { points_.clear(); } size_t size() const { return points_.size(); } Interval domain() const; diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp index cffaa06a..51c49ba9 100644 --- a/src/ipa/rkisp1/algorithms/awb.cpp +++ b/src/ipa/rkisp1/algorithms/awb.cpp @@ -16,6 +16,8 @@ #include <libcamera/ipa/core_ipa_interface.h> +#include "libipa/awb_bayes.h" +#include "libipa/awb_grey.h" #include "libipa/colours.h" /** @@ -40,6 +42,40 @@ constexpr int32_t kDefaultColourTemperature = 5000; /* Minimum mean value below which AWB can't operate. */ constexpr double kMeanMinThreshold = 2.0; +class RkISP1AwbStats : public AwbStats +{ +public: + RkISP1AwbStats(const RGB<double> &rgbMeans) + : rgbMeans_(rgbMeans) + { + rg_ = rgbMeans_.r() / rgbMeans_.g(); + bg_ = rgbMeans_.b() / rgbMeans_.g(); + } + + double computeColourError(const RGB<double> &gains) const override + { + /* + * Compute the sum of the squared colour error (non-greyness) as it + * appears in the log likelihood equation. + */ + double deltaR = gains.r() * rg_ - 1.0; + double deltaB = gains.b() * bg_ - 1.0; + double delta2 = deltaR * deltaR + deltaB * deltaB; + + return delta2; + } + + RGB<double> getRGBMeans() const override + { + return rgbMeans_; + } + +private: + RGB<double> rgbMeans_; + double rg_; + double bg_; +}; + Awb::Awb() : rgbMode_(false) { @@ -55,15 +91,29 @@ int Awb::init(IPAContext &context, const YamlObject &tuningData) kMaxColourTemperature, kDefaultColourTemperature); - Interpolator<Vector<double, 2>> gainCurve; - int ret = gainCurve.readYaml(tuningData["colourGains"], "ct", "gains"); - if (ret < 0) - LOG(RkISP1Awb, Warning) - << "Failed to parse 'colourGains' " - << "parameter from tuning file; " - << "manual colour temperature will not work properly"; - else - colourGainCurve_ = gainCurve; + if (!tuningData.contains("algorithm")) + LOG(RkISP1Awb, Info) << "No awb algorithm specified." + << " Default to grey world"; + + auto mode = tuningData["algorithm"].get<std::string>("grey"); + if (mode == "grey") { + awbAlgo_ = std::make_unique<AwbGrey>(); + } else if (mode == "bayes") { + awbAlgo_ = std::make_unique<AwbBayes>(); + } else { + LOG(RkISP1Awb, Error) << "Unknown awb algorithm: " << mode; + return -EINVAL; + } + LOG(RkISP1Awb, Debug) << "Using awb algorithm: " << mode; + + int ret = awbAlgo_->init(tuningData); + if (ret) { + LOG(RkISP1Awb, Error) << "Failed to init awb algorithm"; + return ret; + } + + const auto &src = awbAlgo_->controls(); + cmap.insert(src.begin(), src.end()); return 0; } @@ -75,7 +125,8 @@ int Awb::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) { context.activeState.awb.gains.manual = RGB<double>{ 1.0 }; - context.activeState.awb.gains.automatic = RGB<double>{ 1.0 }; + context.activeState.awb.gains.automatic = + awbAlgo_->gainsFromColourTemperature(kDefaultColourTemperature); context.activeState.awb.autoEnabled = true; context.activeState.awb.temperatureK = kDefaultColourTemperature; @@ -111,6 +162,8 @@ void Awb::queueRequest(IPAContext &context, << (*awbEnable ? "Enabling" : "Disabling") << " AWB"; } + awbAlgo_->handleControls(controls); + frameContext.awb.autoEnabled = awb.autoEnabled; if (awb.autoEnabled) @@ -128,10 +181,10 @@ void Awb::queueRequest(IPAContext &context, * This will be fixed with the bayes AWB algorithm. */ update = true; - } else if (colourTemperature && colourGainCurve_) { - const auto &gains = colourGainCurve_->getInterpolated(*colourTemperature); - awb.gains.manual.r() = gains[0]; - awb.gains.manual.b() = gains[1]; + } else if (colourTemperature) { + const auto &gains = awbAlgo_->gainsFromColourTemperature(*colourTemperature); + awb.gains.manual.r() = gains.r(); + awb.gains.manual.b() = gains.b(); awb.temperatureK = *colourTemperature; update = true; } @@ -229,7 +282,7 @@ void Awb::process(IPAContext &context, const rkisp1_cif_isp_stat *params = &stats->params; const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; IPAActiveState &activeState = context.activeState; - RGB<double> rgbMeans; + RGB<double> rgbMeans = calculateRgbMeans(frameContext, awb); metadata.set(controls::AwbEnable, frameContext.awb.autoEnabled); metadata.set(controls::ColourGains, { @@ -243,6 +296,48 @@ void Awb::process(IPAContext &context, return; } + /* + * If the means are too small we don't have enough information to + * meaningfully calculate gains. Freeze the algorithm in that case. + */ + if (rgbMeans.r() < kMeanMinThreshold && rgbMeans.g() < kMeanMinThreshold && + rgbMeans.b() < kMeanMinThreshold) + return; + + RkISP1AwbStats awbStats{ rgbMeans }; + AwbResult awbResult = awbAlgo_->calculateAwb(awbStats, frameContext.lux.lux); + + activeState.awb.temperatureK = awbResult.colourTemperature; + + /* Metadata shall contain the up to date measurement */ + metadata.set(controls::ColourTemperature, activeState.awb.temperatureK); + + /* + * Clamp the gain values to the hardware, which expresses gains as Q2.8 + * unsigned integer values. Set the minimum just above zero to avoid + * divisions by zero when computing the raw means in subsequent + * iterations. + */ + awbResult.gains = awbResult.gains.max(1.0 / 256).min(1023.0 / 256); + + /* Filter the values to avoid oscillations. */ + double speed = 0.2; + awbResult.gains = awbResult.gains * speed + + activeState.awb.gains.automatic * (1 - speed); + + activeState.awb.gains.automatic = awbResult.gains; + + LOG(RkISP1Awb, Debug) + << std::showpoint + << "Means " << rgbMeans << ", gains " + << activeState.awb.gains.automatic << ", temp " + << activeState.awb.temperatureK << "K"; +} + +RGB<double> Awb::calculateRgbMeans(const IPAFrameContext &frameContext, const rkisp1_cif_isp_awb_stat *awb) const +{ + Vector<double, 3> rgbMeans; + if (rgbMode_) { rgbMeans = {{ static_cast<double>(awb->awb_mean[0].mean_y_or_g), @@ -301,46 +396,7 @@ void Awb::process(IPAContext &context, */ rgbMeans /= frameContext.awb.gains; - /* - * If the means are too small we don't have enough information to - * meaningfully calculate gains. Freeze the algorithm in that case. - */ - if (rgbMeans.r() < kMeanMinThreshold && rgbMeans.g() < kMeanMinThreshold && - rgbMeans.b() < kMeanMinThreshold) - return; - - activeState.awb.temperatureK = estimateCCT(rgbMeans); - - /* - * Estimate the red and blue gains to apply in a grey world. The green - * gain is hardcoded to 1.0. Avoid divisions by zero by clamping the - * divisor to a minimum value of 1.0. - */ - RGB<double> gains({ - rgbMeans.g() / std::max(rgbMeans.r(), 1.0), - 1.0, - rgbMeans.g() / std::max(rgbMeans.b(), 1.0) - }); - - /* - * Clamp the gain values to the hardware, which expresses gains as Q2.8 - * unsigned integer values. Set the minimum just above zero to avoid - * divisions by zero when computing the raw means in subsequent - * iterations. - */ - gains = gains.max(1.0 / 256).min(1023.0 / 256); - - /* Filter the values to avoid oscillations. */ - double speed = 0.2; - gains = gains * speed + activeState.awb.gains.automatic * (1 - speed); - - activeState.awb.gains.automatic = gains; - - LOG(RkISP1Awb, Debug) - << std::showpoint - << "Means " << rgbMeans << ", gains " - << activeState.awb.gains.automatic << ", temp " - << activeState.awb.temperatureK << "K"; + return rgbMeans; } REGISTER_IPA_ALGORITHM(Awb, "Awb") diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h index 34ec42cb..7e6c3862 100644 --- a/src/ipa/rkisp1/algorithms/awb.h +++ b/src/ipa/rkisp1/algorithms/awb.h @@ -11,6 +11,7 @@ #include "libcamera/internal/vector.h" +#include "libipa/awb.h" #include "libipa/interpolator.h" #include "algorithm.h" @@ -39,7 +40,11 @@ public: ControlList &metadata) override; private: - std::optional<Interpolator<Vector<double, 2>>> colourGainCurve_; + RGB<double> calculateRgbMeans(const IPAFrameContext &frameContext, + const rkisp1_cif_isp_awb_stat *awb) const; + + std::unique_ptr<AwbAlgorithm> awbAlgo_; + bool rgbMode_; }; diff --git a/src/ipa/rkisp1/algorithms/lux.cpp b/src/ipa/rkisp1/algorithms/lux.cpp index b0f74963..a467767e 100644 --- a/src/ipa/rkisp1/algorithms/lux.cpp +++ b/src/ipa/rkisp1/algorithms/lux.cpp @@ -33,12 +33,8 @@ namespace ipa::rkisp1::algorithms { /** * \brief Construct an rkisp1 Lux algo module - * - * The Lux helper is initialized to 65535 as that is the max bin count on the - * rkisp1. */ Lux::Lux() - : lux_(65535) { } diff --git a/src/libcamera/ipa_proxy.cpp b/src/libcamera/ipa_proxy.cpp index 25f772a4..9907b961 100644 --- a/src/libcamera/ipa_proxy.cpp +++ b/src/libcamera/ipa_proxy.cpp @@ -115,7 +115,7 @@ std::string IPAProxy::configurationFile(const std::string &name, ipaEnvName = "LIBCAMERA_" + ipaEnvName + "_TUNING_FILE"; char const *configFromEnv = utils::secure_getenv(ipaEnvName.c_str()); - if (configFromEnv && *configFromEnv == '\0') + if (configFromEnv && *configFromEnv != '\0') return { configFromEnv }; struct stat statbuf; diff --git a/src/libcamera/request.cpp b/src/libcamera/request.cpp index 8c56ed30..b206ac13 100644 --- a/src/libcamera/request.cpp +++ b/src/libcamera/request.cpp @@ -475,6 +475,15 @@ int Request::addBuffer(const Stream *stream, FrameBuffer *buffer, return -EINVAL; } + /* + * Make sure the fence has been extracted from the buffer + * to avoid waiting on a stale fence. + */ + if (buffer->_d()->fence()) { + LOG(Request, Error) << "Can't add buffer that still references a fence"; + return -EEXIST; + } + auto it = bufferMap_.find(stream); if (it != bufferMap_.end()) { LOG(Request, Error) << "FrameBuffer already set for stream"; @@ -485,15 +494,6 @@ int Request::addBuffer(const Stream *stream, FrameBuffer *buffer, _d()->pending_.insert(buffer); bufferMap_[stream] = buffer; - /* - * Make sure the fence has been extracted from the buffer - * to avoid waiting on a stale fence. - */ - if (buffer->_d()->fence()) { - LOG(Request, Error) << "Can't add buffer that still references a fence"; - return -EEXIST; - } - if (fence && fence->isValid()) buffer->_d()->setFence(std::move(fence)); diff --git a/utils/tuning/config-example.yaml b/utils/tuning/config-example.yaml index 1b7f52cd..5593eaef 100644 --- a/utils/tuning/config-example.yaml +++ b/utils/tuning/config-example.yaml @@ -5,7 +5,49 @@ general: do_alsc_colour: 1 luminance_strength: 0.5 awb: - greyworld: 0 + # Algorithm can either be 'grey' or 'bayes' + algorithm: bayes + # Priors is only used for the bayes algorithm. They are defined in linear + # space. A good staring point is: + # - lux: 0 + # ct: [ 2000, 3000, 13000 ] + # probability: [ 1.005, 1.0, 1.0 ] + # - lux: 800 + # ct: [ 2000, 6000, 13000 ] + # probability: [ 1.0, 1.01, 1.01 ] + # - lux: 1500 + # ct: [ 2000, 4000, 6000, 6500, 7000, 13000 ] + # probability: [ 1.0, 1.005, 1.032, 1.037, 1.01, 1.01 ] + priors: + - lux: 0 + ct: [ 2000, 13000 ] + probability: [ 1.0, 1.0 ] + AwbMode: + AwbAuto: + lo: 2500 + hi: 8000 + AwbIncandescent: + lo: 2500 + hi: 3000 + AwbTungsten: + lo: 3000 + hi: 3500 + AwbFluorescent: + lo: 4000 + hi: 4700 + AwbIndoor: + lo: 3000 + hi: 5000 + AwbDaylight: + lo: 5500 + hi: 6500 + AwbCloudy: + lo: 6500 + hi: 8000 + # One custom mode can be defined if needed + #AwbCustom: + # lo: 2000 + # hi: 1300 macbeth: small: 1 show: 0 diff --git a/utils/tuning/libtuning/modules/awb/awb.py b/utils/tuning/libtuning/modules/awb/awb.py index c154cf3b..0dc4f59d 100644 --- a/utils/tuning/libtuning/modules/awb/awb.py +++ b/utils/tuning/libtuning/modules/awb/awb.py @@ -27,10 +27,14 @@ class AWB(Module): imgs = [img for img in images if img.macbeth is not None] - gains, _, _ = awb(imgs, None, None, False) - gains = np.reshape(gains, (-1, 3)) + ct_curve, transverse_pos, transverse_neg = awb(imgs, None, None, False) + ct_curve = np.reshape(ct_curve, (-1, 3)) + gains = [{ + 'ct': int(v[0]), + 'gains': [float(1.0 / v[1]), float(1.0 / v[2])] + } for v in ct_curve] + + return {'colourGains': gains, + 'transversePos': transverse_pos, + 'transverseNeg': transverse_neg} - return [{ - 'ct': int(v[0]), - 'gains': [float(1.0 / v[1]), float(1.0 / v[2])] - } for v in gains] diff --git a/utils/tuning/libtuning/modules/awb/rkisp1.py b/utils/tuning/libtuning/modules/awb/rkisp1.py index 0c95843b..d562d26e 100644 --- a/utils/tuning/libtuning/modules/awb/rkisp1.py +++ b/utils/tuning/libtuning/modules/awb/rkisp1.py @@ -6,9 +6,6 @@ from .awb import AWB -import libtuning as lt - - class AWBRkISP1(AWB): hr_name = 'AWB (RkISP1)' out_name = 'Awb' @@ -20,8 +17,20 @@ class AWBRkISP1(AWB): return True def process(self, config: dict, images: list, outputs: dict) -> dict: - output = {} - - output['colourGains'] = self.do_calculation(images) + if not 'awb' in config['general']: + raise ValueError('AWB configuration missing') + awb_config = config['general']['awb'] + algorithm = awb_config['algorithm'] + + output = {'algorithm': algorithm} + data = self.do_calculation(images) + if algorithm == 'grey': + output['colourGains'] = data['colourGains'] + elif algorithm == 'bayes': + output['AwbMode'] = awb_config['AwbMode'] + output['priors'] = awb_config['priors'] + output.update(data) + else: + raise ValueError(f"Unknown AWB algorithm {output['algorithm']}") return output diff --git a/utils/tuning/libtuning/modules/lux/__init__.py b/utils/tuning/libtuning/modules/lux/__init__.py new file mode 100644 index 00000000..af9d4e08 --- /dev/null +++ b/utils/tuning/libtuning/modules/lux/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2025, Ideas on Board + +from libtuning.modules.lux.lux import Lux +from libtuning.modules.lux.rkisp1 import LuxRkISP1 diff --git a/utils/tuning/libtuning/modules/lux/lux.py b/utils/tuning/libtuning/modules/lux/lux.py new file mode 100644 index 00000000..4bad429a --- /dev/null +++ b/utils/tuning/libtuning/modules/lux/lux.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2019, Raspberry Pi Ltd +# Copyright (C) 2025, Ideas on Board +# +# Base Lux tuning module + +from ..module import Module + +import logging +import numpy as np + +logger = logging.getLogger(__name__) + + +class Lux(Module): + type = 'lux' + hr_name = 'Lux (Base)' + out_name = 'GenericLux' + + def __init__(self, debug: list): + super().__init__() + + self.debug = debug + + def calculate_lux_reference_values(self, images): + # The lux calibration is done on a single image. For best effects, the + # image with lux level closest to 1000 is chosen. + imgs = [img for img in images if img.macbeth is not None] + lux_values = [img.lux for img in imgs] + index = lux_values.index(min(lux_values, key=lambda l: abs(1000 - l))) + img = imgs[index] + logger.info(f'Selected image {img.name} for lux calibration') + + if img.lux < 50: + logger.warning(f'A Lux level of {img.lux} is very low for proper lux calibration') + + ref_y = self.calculate_y(img) + exposure_time = img.exposure + gain = img.againQ8_norm + aperture = 1 + logger.info(f'RefY:{ref_y} Exposure time:{exposure_time}µs Gain:{gain} Aperture:{aperture}') + return {'referenceY': ref_y, + 'referenceExposureTime': exposure_time, + 'referenceAnalogueGain': gain, + 'referenceDigitalGain': 1.0, + 'referenceLux': img.lux} + + def calculate_y(self, img): + max16Bit = 0xffff + # Average over all grey patches. + ap_r = np.mean(img.patches[0][3::4]) / max16Bit + ap_g = (np.mean(img.patches[1][3::4]) + np.mean(img.patches[2][3::4])) / 2 / max16Bit + ap_b = np.mean(img.patches[3][3::4]) / max16Bit + logger.debug(f'Averaged grey patches: Red: {ap_r}, Green: {ap_g}, Blue: {ap_b}') + + # Calculate white balance gains. + gr = ap_g / ap_r + gb = ap_g / ap_b + logger.debug(f'WB gains: Red: {gr} Blue: {gb}') + + # Calculate the mean Y value of the whole image + a_r = np.mean(img.channels[0]) * gr + a_g = (np.mean(img.channels[1]) + np.mean(img.channels[2])) / 2 + a_b = np.mean(img.channels[3]) * gb + y = 0.299 * a_r + 0.587 * a_g + 0.114 * a_b + y /= max16Bit + + return y + diff --git a/utils/tuning/libtuning/modules/lux/rkisp1.py b/utils/tuning/libtuning/modules/lux/rkisp1.py new file mode 100644 index 00000000..62d3f94c --- /dev/null +++ b/utils/tuning/libtuning/modules/lux/rkisp1.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2024, Ideas on Board +# +# Lux module for tuning rkisp1 + +from .lux import Lux + + +class LuxRkISP1(Lux): + hr_name = 'Lux (RkISP1)' + out_name = 'Lux' + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + # We don't need anything from the config file. + def validate_config(self, config: dict) -> bool: + return True + + def process(self, config: dict, images: list, outputs: dict) -> dict: + return self.calculate_lux_reference_values(images) diff --git a/utils/tuning/rkisp1.py b/utils/tuning/rkisp1.py index 9f40fd8b..207b717a 100755 --- a/utils/tuning/rkisp1.py +++ b/utils/tuning/rkisp1.py @@ -6,18 +6,19 @@ # # Tuning script for rkisp1 -import coloredlogs import logging import sys +import coloredlogs import libtuning as lt -from libtuning.parsers import YamlParser from libtuning.generators import YamlOutput -from libtuning.modules.lsc import LSCRkISP1 from libtuning.modules.agc import AGCRkISP1 from libtuning.modules.awb import AWBRkISP1 from libtuning.modules.ccm import CCMRkISP1 +from libtuning.modules.lsc import LSCRkISP1 +from libtuning.modules.lux import LuxRkISP1 from libtuning.modules.static import StaticModule +from libtuning.parsers import YamlParser coloredlogs.install(level=logging.INFO, fmt='%(name)s %(levelname)s %(message)s') @@ -45,12 +46,15 @@ lsc = LSCRkISP1(debug=[lt.Debug.Plot], # This is the function that will be used to smooth the color ratio # values. This can also be a custom function. smoothing_function=lt.smoothing.MedianBlur(3),) +lux = LuxRkISP1(debug=[lt.Debug.Plot]) tuner = lt.Tuner('RkISP1') -tuner.add([agc, awb, blc, ccm, color_processing, filter, gamma_out, lsc]) +tuner.add([agc, awb, blc, ccm, color_processing, filter, gamma_out, lsc, lux]) tuner.set_input_parser(YamlParser()) tuner.set_output_formatter(YamlOutput()) -tuner.set_output_order([agc, awb, blc, ccm, color_processing, + +# Bayesian AWB uses the lux value, so insert the lux algorithm before AWB. +tuner.set_output_order([agc, lux, awb, blc, ccm, color_processing, filter, gamma_out, lsc]) if __name__ == '__main__': |