diff options
Diffstat (limited to 'src/ipa/rkisp1/algorithms')
| -rw-r--r-- | src/ipa/rkisp1/algorithms/agc.cpp | 112 | ||||
| -rw-r--r-- | src/ipa/rkisp1/algorithms/agc.h | 4 | ||||
| -rw-r--r-- | src/ipa/rkisp1/algorithms/awb.cpp | 176 | ||||
| -rw-r--r-- | src/ipa/rkisp1/algorithms/awb.h | 10 | ||||
| -rw-r--r-- | src/ipa/rkisp1/algorithms/ccm.cpp | 7 | ||||
| -rw-r--r-- | src/ipa/rkisp1/algorithms/lux.cpp | 4 |
6 files changed, 204 insertions, 109 deletions
diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp index 1680669c..b3ac9400 100644 --- a/src/ipa/rkisp1/algorithms/agc.cpp +++ b/src/ipa/rkisp1/algorithms/agc.cpp @@ -149,13 +149,13 @@ int Agc::init(IPAContext &context, const YamlObject &tuningData) return ret; context.ctrlMap[&controls::ExposureTimeMode] = - ControlInfo(static_cast<int32_t>(controls::ExposureTimeModeAuto), - static_cast<int32_t>(controls::ExposureTimeModeManual), - static_cast<int32_t>(controls::ExposureTimeModeAuto)); + ControlInfo({ { ControlValue(controls::ExposureTimeModeAuto), + ControlValue(controls::ExposureTimeModeManual) } }, + ControlValue(controls::ExposureTimeModeAuto)); context.ctrlMap[&controls::AnalogueGainMode] = - ControlInfo(static_cast<int32_t>(controls::AnalogueGainModeAuto), - static_cast<int32_t>(controls::AnalogueGainModeManual), - static_cast<int32_t>(controls::AnalogueGainModeAuto)); + ControlInfo({ { ControlValue(controls::AnalogueGainModeAuto), + ControlValue(controls::AnalogueGainModeManual) } }, + ControlValue(controls::AnalogueGainModeAuto)); /* \todo Move this to the Camera class */ context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap.merge(controls()); @@ -188,21 +188,15 @@ int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) context.activeState.agc.meteringMode = static_cast<controls::AeMeteringModeEnum>(meteringModes_.begin()->first); - /* - * \todo This should probably come from FrameDurationLimits instead, - * except it's computed in the IPA and not here so we'd have to - * recompute it. - */ - context.activeState.agc.maxFrameDuration = context.configuration.sensor.maxExposureTime; + /* Limit the frame duration to match current initialisation */ + ControlInfo &frameDurationLimits = context.ctrlMap[&controls::FrameDurationLimits]; + context.activeState.agc.minFrameDuration = std::chrono::microseconds(frameDurationLimits.min().get<int64_t>()); + context.activeState.agc.maxFrameDuration = std::chrono::microseconds(frameDurationLimits.max().get<int64_t>()); - /* - * Define the measurement window for AGC as a centered rectangle - * covering 3/4 of the image width and height. - */ - context.configuration.agc.measureWindow.h_offs = configInfo.outputSize.width / 8; - context.configuration.agc.measureWindow.v_offs = configInfo.outputSize.height / 8; - context.configuration.agc.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; - context.configuration.agc.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; + context.configuration.agc.measureWindow.h_offs = 0; + context.configuration.agc.measureWindow.v_offs = 0; + context.configuration.agc.measureWindow.h_size = configInfo.outputSize.width; + context.configuration.agc.measureWindow.v_size = configInfo.outputSize.height; setLimits(context.configuration.sensor.minExposureTime, context.configuration.sensor.maxExposureTime, @@ -310,10 +304,21 @@ void Agc::queueRequest(IPAContext &context, const auto &frameDurationLimits = controls.get(controls::FrameDurationLimits); if (frameDurationLimits) { - utils::Duration maxFrameDuration = - std::chrono::milliseconds((*frameDurationLimits).back()); - agc.maxFrameDuration = maxFrameDuration; + /* Limit the control value to the limits in ControlInfo */ + ControlInfo &limits = context.ctrlMap[&controls::FrameDurationLimits]; + int64_t minFrameDuration = + std::clamp((*frameDurationLimits).front(), + limits.min().get<int64_t>(), + limits.max().get<int64_t>()); + int64_t maxFrameDuration = + std::clamp((*frameDurationLimits).back(), + limits.min().get<int64_t>(), + limits.max().get<int64_t>()); + + agc.minFrameDuration = std::chrono::microseconds(minFrameDuration); + agc.maxFrameDuration = std::chrono::microseconds(maxFrameDuration); } + frameContext.agc.minFrameDuration = agc.minFrameDuration; frameContext.agc.maxFrameDuration = agc.maxFrameDuration; } @@ -390,6 +395,7 @@ void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, * frameContext.sensor.exposure; metadata.set(controls::AnalogueGain, frameContext.sensor.gain); metadata.set(controls::ExposureTime, exposureTime.get<std::micro>()); + metadata.set(controls::FrameDuration, frameContext.agc.frameDuration.get<std::micro>()); metadata.set(controls::ExposureTimeMode, frameContext.agc.autoExposureEnabled ? controls::ExposureTimeModeAuto @@ -399,13 +405,6 @@ void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, ? controls::AnalogueGainModeAuto : controls::AnalogueGainModeManual); - /* \todo Use VBlank value calculated from each frame exposure. */ - uint32_t vTotal = context.configuration.sensor.size.height - + context.configuration.sensor.defVBlank; - utils::Duration frameDuration = context.configuration.sensor.lineDuration - * vTotal; - metadata.set(controls::FrameDuration, frameDuration.get<std::micro>()); - metadata.set(controls::AeMeteringMode, frameContext.agc.meteringMode); metadata.set(controls::AeExposureMode, frameContext.agc.exposureMode); metadata.set(controls::AeConstraintMode, frameContext.agc.constraintMode); @@ -436,15 +435,41 @@ void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, */ double Agc::estimateLuminance(double gain) const { + ASSERT(expMeans_.size() == weights_.size()); double ySum = 0.0; + double wSum = 0.0; /* Sum the averages, saturated to 255. */ - for (uint8_t expMean : expMeans_) - ySum += std::min(expMean * gain, 255.0); + for (unsigned i = 0; i < expMeans_.size(); i++) { + double w = weights_[i]; + ySum += std::min(expMeans_[i] * gain, 255.0) * w; + wSum += w; + } /* \todo Weight with the AWB gains */ - return ySum / expMeans_.size() / 255; + return ySum / wSum / 255; +} + +/** + * \brief Process frame duration and compute vblank + * \param[in] context The shared IPA context + * \param[in] frameContext The current frame context + * \param[in] frameDuration The target frame duration + * + * Compute and populate vblank from the target frame duration. + */ +void Agc::processFrameDuration(IPAContext &context, + IPAFrameContext &frameContext, + utils::Duration frameDuration) +{ + IPACameraSensorInfo &sensorInfo = context.sensorInfo; + utils::Duration lineDuration = context.configuration.sensor.lineDuration; + + frameContext.agc.vblank = (frameDuration / lineDuration) - sensorInfo.outputSize.height; + + /* Update frame duration accounting for line length quantization. */ + frameContext.agc.frameDuration = (sensorInfo.outputSize.height + frameContext.agc.vblank) * lineDuration; } /** @@ -463,16 +488,20 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, ControlList &metadata) { if (!stats) { + processFrameDuration(context, frameContext, + frameContext.agc.minFrameDuration); fillMetadata(context, frameContext, metadata); return; } - + if (!(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP)) { fillMetadata(context, frameContext, metadata); LOG(RkISP1Agc, Error) << "AUTOEXP data is missing in statistics"; return; } + const utils::Duration &lineDuration = context.configuration.sensor.lineDuration; + /* * \todo Verify that the exposure and gain applied by the sensor for * this frame match what has been requested. This isn't a hard @@ -487,6 +516,8 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, Histogram hist({ params->hist.hist_bins, context.hw->numHistogramBins }, [](uint32_t x) { return x >> 4; }); expMeans_ = { params->ae.exp_mean, context.hw->numAeCells }; + std::vector<uint8_t> &modeWeights = meteringModes_.at(frameContext.agc.meteringMode); + weights_ = { modeWeights.data(), modeWeights.size() }; /* * Set the AGC limits using the fixed exposure time and/or gain in @@ -522,8 +553,7 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, * The Agc algorithm needs to know the effective exposure value that was * applied to the sensor when the statistics were collected. */ - utils::Duration exposureTime = context.configuration.sensor.lineDuration - * frameContext.sensor.exposure; + utils::Duration exposureTime = lineDuration * frameContext.sensor.exposure; double analogueGain = frameContext.sensor.gain; utils::Duration effectiveExposureValue = exposureTime * analogueGain; @@ -540,10 +570,16 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, IPAActiveState &activeState = context.activeState; /* Update the estimated exposure and gain. */ - activeState.agc.automatic.exposure = newExposureTime - / context.configuration.sensor.lineDuration; + activeState.agc.automatic.exposure = newExposureTime / lineDuration; activeState.agc.automatic.gain = aGain; + /* + * Expand the target frame duration so that we do not run faster than + * the minimum frame duration when we have short exposures. + */ + processFrameDuration(context, frameContext, + std::max(frameContext.agc.minFrameDuration, newExposureTime)); + fillMetadata(context, frameContext, metadata); expMeans_ = {}; } diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h index aa86f2c5..7867eed9 100644 --- a/src/ipa/rkisp1/algorithms/agc.h +++ b/src/ipa/rkisp1/algorithms/agc.h @@ -50,8 +50,12 @@ private: void fillMetadata(IPAContext &context, IPAFrameContext &frameContext, ControlList &metadata); double estimateLuminance(double gain) const override; + void processFrameDuration(IPAContext &context, + IPAFrameContext &frameContext, + utils::Duration frameDuration); Span<const uint8_t> expMeans_; + Span<const uint8_t> weights_; std::map<int32_t, std::vector<uint8_t>> meteringModes_; }; diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp index cffaa06a..eafe9308 100644 --- a/src/ipa/rkisp1/algorithms/awb.cpp +++ b/src/ipa/rkisp1/algorithms/awb.cpp @@ -16,6 +16,8 @@ #include <libcamera/ipa/core_ipa_interface.h> +#include "libipa/awb_bayes.h" +#include "libipa/awb_grey.h" #include "libipa/colours.h" /** @@ -40,6 +42,40 @@ constexpr int32_t kDefaultColourTemperature = 5000; /* Minimum mean value below which AWB can't operate. */ constexpr double kMeanMinThreshold = 2.0; +class RkISP1AwbStats final : public AwbStats +{ +public: + RkISP1AwbStats(const RGB<double> &rgbMeans) + : rgbMeans_(rgbMeans) + { + rg_ = rgbMeans_.r() / rgbMeans_.g(); + bg_ = rgbMeans_.b() / rgbMeans_.g(); + } + + double computeColourError(const RGB<double> &gains) const override + { + /* + * Compute the sum of the squared colour error (non-greyness) as + * it appears in the log likelihood equation. + */ + double deltaR = gains.r() * rg_ - 1.0; + double deltaB = gains.b() * bg_ - 1.0; + double delta2 = deltaR * deltaR + deltaB * deltaB; + + return delta2; + } + + RGB<double> rgbMeans() const override + { + return rgbMeans_; + } + +private: + RGB<double> rgbMeans_; + double rg_; + double bg_; +}; + Awb::Awb() : rgbMode_(false) { @@ -55,15 +91,29 @@ int Awb::init(IPAContext &context, const YamlObject &tuningData) kMaxColourTemperature, kDefaultColourTemperature); - Interpolator<Vector<double, 2>> gainCurve; - int ret = gainCurve.readYaml(tuningData["colourGains"], "ct", "gains"); - if (ret < 0) - LOG(RkISP1Awb, Warning) - << "Failed to parse 'colourGains' " - << "parameter from tuning file; " - << "manual colour temperature will not work properly"; - else - colourGainCurve_ = gainCurve; + if (!tuningData.contains("algorithm")) + LOG(RkISP1Awb, Info) << "No AWB algorithm specified." + << " Default to grey world"; + + auto mode = tuningData["algorithm"].get<std::string>("grey"); + if (mode == "grey") { + awbAlgo_ = std::make_unique<AwbGrey>(); + } else if (mode == "bayes") { + awbAlgo_ = std::make_unique<AwbBayes>(); + } else { + LOG(RkISP1Awb, Error) << "Unknown AWB algorithm: " << mode; + return -EINVAL; + } + LOG(RkISP1Awb, Debug) << "Using AWB algorithm: " << mode; + + int ret = awbAlgo_->init(tuningData); + if (ret) { + LOG(RkISP1Awb, Error) << "Failed to init AWB algorithm"; + return ret; + } + + const auto &src = awbAlgo_->controls(); + cmap.insert(src.begin(), src.end()); return 0; } @@ -75,7 +125,8 @@ int Awb::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) { context.activeState.awb.gains.manual = RGB<double>{ 1.0 }; - context.activeState.awb.gains.automatic = RGB<double>{ 1.0 }; + context.activeState.awb.gains.automatic = + awbAlgo_->gainsFromColourTemperature(kDefaultColourTemperature); context.activeState.awb.autoEnabled = true; context.activeState.awb.temperatureK = kDefaultColourTemperature; @@ -111,6 +162,8 @@ void Awb::queueRequest(IPAContext &context, << (*awbEnable ? "Enabling" : "Disabling") << " AWB"; } + awbAlgo_->handleControls(controls); + frameContext.awb.autoEnabled = awb.autoEnabled; if (awb.autoEnabled) @@ -123,15 +176,15 @@ void Awb::queueRequest(IPAContext &context, awb.gains.manual.r() = (*colourGains)[0]; awb.gains.manual.b() = (*colourGains)[1]; /* - * \todo: Colour temperature reported in metadata is now + * \todo Colour temperature reported in metadata is now * incorrect, as we can't deduce the temperature from the gains. * This will be fixed with the bayes AWB algorithm. */ update = true; - } else if (colourTemperature && colourGainCurve_) { - const auto &gains = colourGainCurve_->getInterpolated(*colourTemperature); - awb.gains.manual.r() = gains[0]; - awb.gains.manual.b() = gains[1]; + } else if (colourTemperature) { + const auto &gains = awbAlgo_->gainsFromColourTemperature(*colourTemperature); + awb.gains.manual.r() = gains.r(); + awb.gains.manual.b() = gains.b(); awb.temperatureK = *colourTemperature; update = true; } @@ -226,10 +279,7 @@ void Awb::process(IPAContext &context, const rkisp1_stat_buffer *stats, ControlList &metadata) { - const rkisp1_cif_isp_stat *params = &stats->params; - const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; IPAActiveState &activeState = context.activeState; - RGB<double> rgbMeans; metadata.set(controls::AwbEnable, frameContext.awb.autoEnabled); metadata.set(controls::ColourGains, { @@ -243,10 +293,57 @@ void Awb::process(IPAContext &context, return; } + const rkisp1_cif_isp_stat *params = &stats->params; + const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; + + RGB<double> rgbMeans = calculateRgbMeans(frameContext, awb); + + /* + * If the means are too small we don't have enough information to + * meaningfully calculate gains. Freeze the algorithm in that case. + */ + if (rgbMeans.r() < kMeanMinThreshold && rgbMeans.g() < kMeanMinThreshold && + rgbMeans.b() < kMeanMinThreshold) + return; + + RkISP1AwbStats awbStats{ rgbMeans }; + AwbResult awbResult = awbAlgo_->calculateAwb(awbStats, frameContext.lux.lux); + + activeState.awb.temperatureK = awbResult.colourTemperature; + + /* Metadata shall contain the up to date measurement */ + metadata.set(controls::ColourTemperature, activeState.awb.temperatureK); + + /* + * Clamp the gain values to the hardware, which expresses gains as Q2.8 + * unsigned integer values. Set the minimum just above zero to avoid + * divisions by zero when computing the raw means in subsequent + * iterations. + */ + awbResult.gains = awbResult.gains.max(1.0 / 256).min(1023.0 / 256); + + /* Filter the values to avoid oscillations. */ + double speed = 0.2; + awbResult.gains = awbResult.gains * speed + + activeState.awb.gains.automatic * (1 - speed); + + activeState.awb.gains.automatic = awbResult.gains; + + LOG(RkISP1Awb, Debug) + << std::showpoint + << "Means " << rgbMeans << ", gains " + << activeState.awb.gains.automatic << ", temp " + << activeState.awb.temperatureK << "K"; +} + +RGB<double> Awb::calculateRgbMeans(const IPAFrameContext &frameContext, const rkisp1_cif_isp_awb_stat *awb) const +{ + Vector<double, 3> rgbMeans; + if (rgbMode_) { rgbMeans = {{ - static_cast<double>(awb->awb_mean[0].mean_y_or_g), static_cast<double>(awb->awb_mean[0].mean_cr_or_r), + static_cast<double>(awb->awb_mean[0].mean_y_or_g), static_cast<double>(awb->awb_mean[0].mean_cb_or_b) }}; } else { @@ -301,46 +398,7 @@ void Awb::process(IPAContext &context, */ rgbMeans /= frameContext.awb.gains; - /* - * If the means are too small we don't have enough information to - * meaningfully calculate gains. Freeze the algorithm in that case. - */ - if (rgbMeans.r() < kMeanMinThreshold && rgbMeans.g() < kMeanMinThreshold && - rgbMeans.b() < kMeanMinThreshold) - return; - - activeState.awb.temperatureK = estimateCCT(rgbMeans); - - /* - * Estimate the red and blue gains to apply in a grey world. The green - * gain is hardcoded to 1.0. Avoid divisions by zero by clamping the - * divisor to a minimum value of 1.0. - */ - RGB<double> gains({ - rgbMeans.g() / std::max(rgbMeans.r(), 1.0), - 1.0, - rgbMeans.g() / std::max(rgbMeans.b(), 1.0) - }); - - /* - * Clamp the gain values to the hardware, which expresses gains as Q2.8 - * unsigned integer values. Set the minimum just above zero to avoid - * divisions by zero when computing the raw means in subsequent - * iterations. - */ - gains = gains.max(1.0 / 256).min(1023.0 / 256); - - /* Filter the values to avoid oscillations. */ - double speed = 0.2; - gains = gains * speed + activeState.awb.gains.automatic * (1 - speed); - - activeState.awb.gains.automatic = gains; - - LOG(RkISP1Awb, Debug) - << std::showpoint - << "Means " << rgbMeans << ", gains " - << activeState.awb.gains.automatic << ", temp " - << activeState.awb.temperatureK << "K"; + return rgbMeans; } REGISTER_IPA_ALGORITHM(Awb, "Awb") diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h index e4248048..7e6c3862 100644 --- a/src/ipa/rkisp1/algorithms/awb.h +++ b/src/ipa/rkisp1/algorithms/awb.h @@ -9,8 +9,10 @@ #include <optional> +#include "libcamera/internal/vector.h" + +#include "libipa/awb.h" #include "libipa/interpolator.h" -#include "libipa/vector.h" #include "algorithm.h" @@ -38,7 +40,11 @@ public: ControlList &metadata) override; private: - std::optional<Interpolator<Vector<double, 2>>> colourGainCurve_; + RGB<double> calculateRgbMeans(const IPAFrameContext &frameContext, + const rkisp1_cif_isp_awb_stat *awb) const; + + std::unique_ptr<AwbAlgorithm> awbAlgo_; + bool rgbMode_; }; diff --git a/src/ipa/rkisp1/algorithms/ccm.cpp b/src/ipa/rkisp1/algorithms/ccm.cpp index e2b5cf4d..eb8ca39e 100644 --- a/src/ipa/rkisp1/algorithms/ccm.cpp +++ b/src/ipa/rkisp1/algorithms/ccm.cpp @@ -120,12 +120,7 @@ void Ccm::process([[maybe_unused]] IPAContext &context, [[maybe_unused]] const rkisp1_stat_buffer *stats, ControlList &metadata) { - float m[9]; - for (unsigned int i = 0; i < 3; i++) { - for (unsigned int j = 0; j < 3; j++) - m[i * 3 + j] = frameContext.ccm.ccm[i][j]; - } - metadata.set(controls::ColourCorrectionMatrix, m); + metadata.set(controls::ColourCorrectionMatrix, frameContext.ccm.ccm.data()); } REGISTER_IPA_ALGORITHM(Ccm, "Ccm") diff --git a/src/ipa/rkisp1/algorithms/lux.cpp b/src/ipa/rkisp1/algorithms/lux.cpp index b0f74963..a467767e 100644 --- a/src/ipa/rkisp1/algorithms/lux.cpp +++ b/src/ipa/rkisp1/algorithms/lux.cpp @@ -33,12 +33,8 @@ namespace ipa::rkisp1::algorithms { /** * \brief Construct an rkisp1 Lux algo module - * - * The Lux helper is initialized to 65535 as that is the max bin count on the - * rkisp1. */ Lux::Lux() - : lux_(65535) { } |