diff options
Diffstat (limited to 'src/ipa/rkisp1')
39 files changed, 2488 insertions, 883 deletions
diff --git a/src/ipa/rkisp1/algorithms/agc.cpp b/src/ipa/rkisp1/algorithms/agc.cpp index e5aeb342..137a0750 100644 --- a/src/ipa/rkisp1/algorithms/agc.cpp +++ b/src/ipa/rkisp1/algorithms/agc.cpp @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * agc.cpp - AGC/AEC mean-based control algorithm + * AGC/AEC mean-based control algorithm */ #include "agc.h" @@ -10,6 +10,8 @@ #include <algorithm> #include <chrono> #include <cmath> +#include <tuple> +#include <vector> #include <libcamera/base/log.h> #include <libcamera/base/utils.h> @@ -17,6 +19,8 @@ #include <libcamera/control_ids.h> #include <libcamera/ipa/core_ipa_interface.h> +#include "libcamera/internal/yaml_parser.h" + #include "libipa/histogram.h" /** @@ -36,35 +40,129 @@ namespace ipa::rkisp1::algorithms { LOG_DEFINE_CATEGORY(RkISP1Agc) -/* Minimum limit for analogue gain value */ -static constexpr double kMinAnalogueGain = 1.0; +int Agc::parseMeteringModes(IPAContext &context, const YamlObject &tuningData) +{ + if (!tuningData.isDictionary()) + LOG(RkISP1Agc, Warning) + << "'AeMeteringMode' parameter not found in tuning file"; + + for (const auto &[key, value] : tuningData.asDict()) { + if (controls::AeMeteringModeNameValueMap.find(key) == + controls::AeMeteringModeNameValueMap.end()) { + LOG(RkISP1Agc, Warning) + << "Skipping unknown metering mode '" << key << "'"; + continue; + } -/* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */ -static constexpr utils::Duration kMaxShutterSpeed = 60ms; + std::vector<uint8_t> weights = + value.getList<uint8_t>().value_or(std::vector<uint8_t>{}); + if (weights.size() != context.hw->numHistogramWeights) { + LOG(RkISP1Agc, Warning) + << "Failed to read metering mode'" << key << "'"; + continue; + } -/* Number of frames to wait before calculating stats on minimum exposure */ -static constexpr uint32_t kNumStartupFrames = 10; + meteringModes_[controls::AeMeteringModeNameValueMap.at(key)] = weights; + } -/* Target value to reach for the top 2% of the histogram */ -static constexpr double kEvGainTarget = 0.5; + if (meteringModes_.empty()) { + LOG(RkISP1Agc, Warning) + << "No metering modes read from tuning file; defaulting to matrix"; + std::vector<uint8_t> weights(context.hw->numHistogramWeights, 1); -/* - * Relative luminance target. - * - * It's a number that's chosen so that, when the camera points at a grey - * target, the resulting image brightness is considered right. - * - * \todo Why is the value different between IPU3 and RkISP1 ? - */ -static constexpr double kRelativeLuminanceTarget = 0.4; + meteringModes_[controls::MeteringMatrix] = weights; + } + + std::vector<ControlValue> meteringModes; + std::vector<int> meteringModeKeys = utils::map_keys(meteringModes_); + std::transform(meteringModeKeys.begin(), meteringModeKeys.end(), + std::back_inserter(meteringModes), + [](int x) { return ControlValue(x); }); + context.ctrlMap[&controls::AeMeteringMode] = ControlInfo(meteringModes); + + return 0; +} + +uint8_t Agc::computeHistogramPredivider(const Size &size, + enum rkisp1_cif_isp_histogram_mode mode) +{ + /* + * The maximum number of pixels that could potentially be in one bin is + * if all the pixels of the image are in it, multiplied by 3 for the + * three color channels. The counter for each bin is 16 bits wide, so + * `factor` thus contains the number of times we'd wrap around. This is + * obviously the number of pixels that we need to skip to make sure + * that we don't wrap around, but we compute the square root of it + * instead, as the skip that we need to program is for both the x and y + * directions. + * + * Even though it looks like dividing into a counter of 65536 would + * overflow by 1, this is apparently fine according to the hardware + * documentation, and this successfully gets the expected documented + * predivider size for cases where: + * (width / predivider) * (height / predivider) * 3 == 65536. + * + * There's a bit of extra rounding math to make sure the rounding goes + * the correct direction so that the square of the step is big enough + * to encompass the `factor` number of pixels that we need to skip. + * + * \todo Take into account weights. That is, if the weights are low + * enough we can potentially reduce the predivider to increase + * precision. This needs some investigation however, as this hardware + * behavior is undocumented and is only an educated guess. + */ + int count = mode == RKISP1_CIF_ISP_HISTOGRAM_MODE_RGB_COMBINED ? 3 : 1; + double factor = size.width * size.height * count / 65536.0; + double root = std::sqrt(factor); + uint8_t predivider = static_cast<uint8_t>(std::ceil(root)); + + return std::clamp<uint8_t>(predivider, 3, 127); +} Agc::Agc() - : frameCount_(0), numCells_(0), numHistBins_(0), filteredExposure_(0s) { supportsRaw_ = true; } /** + * \brief Initialise the AGC algorithm from tuning files + * \param[in] context The shared IPA context + * \param[in] tuningData The YamlObject containing Agc tuning data + * + * This function calls the base class' tuningData parsers to discover which + * control values are supported. + * + * \return 0 on success or errors from the base class + */ +int Agc::init(IPAContext &context, const YamlObject &tuningData) +{ + int ret; + + ret = parseTuningData(tuningData); + if (ret) + return ret; + + const YamlObject &yamlMeteringModes = tuningData["AeMeteringMode"]; + ret = parseMeteringModes(context, yamlMeteringModes); + if (ret) + return ret; + + context.ctrlMap[&controls::ExposureTimeMode] = + ControlInfo({ { ControlValue(controls::ExposureTimeModeAuto), + ControlValue(controls::ExposureTimeModeManual) } }, + ControlValue(controls::ExposureTimeModeAuto)); + context.ctrlMap[&controls::AnalogueGainMode] = + ControlInfo({ { ControlValue(controls::AnalogueGainModeAuto), + ControlValue(controls::AnalogueGainModeManual) } }, + ControlValue(controls::AnalogueGainModeAuto)); + /* \todo Move this to the Camera class */ + context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); + context.ctrlMap.merge(controls()); + + return 0; +} + +/** * \brief Configure the AGC given a configInfo * \param[in] context The shared IPA context * \param[in] configInfo The IPA configuration data @@ -79,35 +177,33 @@ int Agc::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) 10ms / context.configuration.sensor.lineDuration; context.activeState.agc.manual.gain = context.activeState.agc.automatic.gain; context.activeState.agc.manual.exposure = context.activeState.agc.automatic.exposure; - context.activeState.agc.autoEnabled = !context.configuration.raw; + context.activeState.agc.autoExposureEnabled = !context.configuration.raw; + context.activeState.agc.autoGainEnabled = !context.configuration.raw; - /* - * According to the RkISP1 documentation: - * - versions < V12 have RKISP1_CIF_ISP_AE_MEAN_MAX_V10 entries, - * - versions >= V12 have RKISP1_CIF_ISP_AE_MEAN_MAX_V12 entries. - */ - if (context.configuration.hw.revision < RKISP1_V12) { - numCells_ = RKISP1_CIF_ISP_AE_MEAN_MAX_V10; - numHistBins_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V10; - } else { - numCells_ = RKISP1_CIF_ISP_AE_MEAN_MAX_V12; - numHistBins_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V12; - } + context.activeState.agc.constraintMode = + static_cast<controls::AeConstraintModeEnum>(constraintModes().begin()->first); + context.activeState.agc.exposureMode = + static_cast<controls::AeExposureModeEnum>(exposureModeHelpers().begin()->first); + context.activeState.agc.meteringMode = + static_cast<controls::AeMeteringModeEnum>(meteringModes_.begin()->first); - /* - * Define the measurement window for AGC as a centered rectangle - * covering 3/4 of the image width and height. - */ - context.configuration.agc.measureWindow.h_offs = configInfo.outputSize.width / 8; - context.configuration.agc.measureWindow.v_offs = configInfo.outputSize.height / 8; - context.configuration.agc.measureWindow.h_size = 3 * configInfo.outputSize.width / 4; - context.configuration.agc.measureWindow.v_size = 3 * configInfo.outputSize.height / 4; + /* Limit the frame duration to match current initialisation */ + ControlInfo &frameDurationLimits = context.ctrlMap[&controls::FrameDurationLimits]; + context.activeState.agc.minFrameDuration = std::chrono::microseconds(frameDurationLimits.min().get<int64_t>()); + context.activeState.agc.maxFrameDuration = std::chrono::microseconds(frameDurationLimits.max().get<int64_t>()); + + context.configuration.agc.measureWindow.h_offs = 0; + context.configuration.agc.measureWindow.v_offs = 0; + context.configuration.agc.measureWindow.h_size = configInfo.outputSize.width; + context.configuration.agc.measureWindow.v_size = configInfo.outputSize.height; + + setLimits(context.configuration.sensor.minExposureTime, + context.configuration.sensor.maxExposureTime, + context.configuration.sensor.minAnalogueGain, + context.configuration.sensor.maxAnalogueGain); + + resetFrameCount(); - /* - * \todo Use the upcoming per-frame context API that will provide a - * frame index - */ - frameCount_ = 0; return 0; } @@ -122,18 +218,47 @@ void Agc::queueRequest(IPAContext &context, auto &agc = context.activeState.agc; if (!context.configuration.raw) { - const auto &agcEnable = controls.get(controls::AeEnable); - if (agcEnable && *agcEnable != agc.autoEnabled) { - agc.autoEnabled = *agcEnable; + const auto &aeEnable = controls.get(controls::ExposureTimeMode); + if (aeEnable && + (*aeEnable == controls::ExposureTimeModeAuto) != agc.autoExposureEnabled) { + agc.autoExposureEnabled = (*aeEnable == controls::ExposureTimeModeAuto); + + LOG(RkISP1Agc, Debug) + << (agc.autoExposureEnabled ? "Enabling" : "Disabling") + << " AGC (exposure)"; + + /* + * If we go from auto -> manual with no manual control + * set, use the last computed value, which we don't + * know until prepare() so save this information. + * + * \todo Check the previous frame at prepare() time + * instead of saving a flag here + */ + if (!agc.autoExposureEnabled && !controls.get(controls::ExposureTime)) + frameContext.agc.autoExposureModeChange = true; + } + + const auto &agEnable = controls.get(controls::AnalogueGainMode); + if (agEnable && + (*agEnable == controls::AnalogueGainModeAuto) != agc.autoGainEnabled) { + agc.autoGainEnabled = (*agEnable == controls::AnalogueGainModeAuto); LOG(RkISP1Agc, Debug) - << (agc.autoEnabled ? "Enabling" : "Disabling") - << " AGC"; + << (agc.autoGainEnabled ? "Enabling" : "Disabling") + << " AGC (gain)"; + /* + * If we go from auto -> manual with no manual control + * set, use the last computed value, which we don't + * know until prepare() so save this information. + */ + if (!agc.autoGainEnabled && !controls.get(controls::AnalogueGain)) + frameContext.agc.autoGainModeChange = true; } } const auto &exposure = controls.get(controls::ExposureTime); - if (exposure && !agc.autoEnabled) { + if (exposure && !agc.autoExposureEnabled) { agc.manual.exposure = *exposure * 1.0us / context.configuration.sensor.lineDuration; @@ -142,183 +267,150 @@ void Agc::queueRequest(IPAContext &context, } const auto &gain = controls.get(controls::AnalogueGain); - if (gain && !agc.autoEnabled) { + if (gain && !agc.autoGainEnabled) { agc.manual.gain = *gain; LOG(RkISP1Agc, Debug) << "Set gain to " << agc.manual.gain; } - frameContext.agc.autoEnabled = agc.autoEnabled; + frameContext.agc.autoExposureEnabled = agc.autoExposureEnabled; + frameContext.agc.autoGainEnabled = agc.autoGainEnabled; - if (!frameContext.agc.autoEnabled) { + if (!frameContext.agc.autoExposureEnabled) frameContext.agc.exposure = agc.manual.exposure; + if (!frameContext.agc.autoGainEnabled) frameContext.agc.gain = agc.manual.gain; + + const auto &meteringMode = controls.get(controls::AeMeteringMode); + if (meteringMode) { + frameContext.agc.updateMetering = agc.meteringMode != *meteringMode; + agc.meteringMode = + static_cast<controls::AeMeteringModeEnum>(*meteringMode); } + frameContext.agc.meteringMode = agc.meteringMode; + + const auto &exposureMode = controls.get(controls::AeExposureMode); + if (exposureMode) + agc.exposureMode = + static_cast<controls::AeExposureModeEnum>(*exposureMode); + frameContext.agc.exposureMode = agc.exposureMode; + + const auto &constraintMode = controls.get(controls::AeConstraintMode); + if (constraintMode) + agc.constraintMode = + static_cast<controls::AeConstraintModeEnum>(*constraintMode); + frameContext.agc.constraintMode = agc.constraintMode; + + const auto &frameDurationLimits = controls.get(controls::FrameDurationLimits); + if (frameDurationLimits) { + /* Limit the control value to the limits in ControlInfo */ + ControlInfo &limits = context.ctrlMap[&controls::FrameDurationLimits]; + int64_t minFrameDuration = + std::clamp((*frameDurationLimits).front(), + limits.min().get<int64_t>(), + limits.max().get<int64_t>()); + int64_t maxFrameDuration = + std::clamp((*frameDurationLimits).back(), + limits.min().get<int64_t>(), + limits.max().get<int64_t>()); + + agc.minFrameDuration = std::chrono::microseconds(minFrameDuration); + agc.maxFrameDuration = std::chrono::microseconds(maxFrameDuration); + } + frameContext.agc.minFrameDuration = agc.minFrameDuration; + frameContext.agc.maxFrameDuration = agc.maxFrameDuration; } /** * \copydoc libcamera::ipa::Algorithm::prepare */ void Agc::prepare(IPAContext &context, const uint32_t frame, - IPAFrameContext &frameContext, rkisp1_params_cfg *params) -{ - if (frameContext.agc.autoEnabled) { - frameContext.agc.exposure = context.activeState.agc.automatic.exposure; - frameContext.agc.gain = context.activeState.agc.automatic.gain; - } - - if (frame > 0) - return; - - /* Configure the measurement window. */ - params->meas.aec_config.meas_window = context.configuration.agc.measureWindow; - /* Use a continuous method for measure. */ - params->meas.aec_config.autostop = RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_0; - /* Estimate Y as (R + G + B) x (85/256). */ - params->meas.aec_config.mode = RKISP1_CIF_ISP_EXP_MEASURING_MODE_1; - - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AEC; - params->module_ens |= RKISP1_CIF_ISP_MODULE_AEC; - params->module_en_update |= RKISP1_CIF_ISP_MODULE_AEC; - - /* Configure histogram. */ - params->meas.hst_config.meas_window = context.configuration.agc.measureWindow; - /* Produce the luminance histogram. */ - params->meas.hst_config.mode = RKISP1_CIF_ISP_HISTOGRAM_MODE_Y_HISTOGRAM; - /* Set an average weighted histogram. */ - for (unsigned int histBin = 0; histBin < numHistBins_; histBin++) - params->meas.hst_config.hist_weight[histBin] = 1; - /* Step size can't be less than 3. */ - params->meas.hst_config.histogram_predivider = 4; - - /* Update the configuration for histogram. */ - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_HST; - /* Enable the histogram measure unit. */ - params->module_ens |= RKISP1_CIF_ISP_MODULE_HST; - params->module_en_update |= RKISP1_CIF_ISP_MODULE_HST; -} - -/** - * \brief Apply a filter on the exposure value to limit the speed of changes - * \param[in] exposureValue The target exposure from the AGC algorithm - * - * The speed of the filter is adaptive, and will produce the target quicker - * during startup, or when the target exposure is within 20% of the most recent - * filter output. - * - * \return The filtered exposure - */ -utils::Duration Agc::filterExposure(utils::Duration exposureValue) + IPAFrameContext &frameContext, RkISP1Params *params) { - double speed = 0.2; + uint32_t activeAutoExposure = context.activeState.agc.automatic.exposure; + double activeAutoGain = context.activeState.agc.automatic.gain; - /* Adapt instantly if we are in startup phase. */ - if (frameCount_ < kNumStartupFrames) - speed = 1.0; + /* Populate exposure and gain in auto mode */ + if (frameContext.agc.autoExposureEnabled) + frameContext.agc.exposure = activeAutoExposure; + if (frameContext.agc.autoGainEnabled) + frameContext.agc.gain = activeAutoGain; /* - * If we are close to the desired result, go faster to avoid making - * multiple micro-adjustments. - * \todo Make this customisable? + * Populate manual exposure and gain from the active auto values when + * transitioning from auto to manual */ - if (filteredExposure_ < 1.2 * exposureValue && - filteredExposure_ > 0.8 * exposureValue) - speed = sqrt(speed); - - filteredExposure_ = speed * exposureValue + - filteredExposure_ * (1.0 - speed); - - LOG(RkISP1Agc, Debug) << "After filtering, exposure " << filteredExposure_; - - return filteredExposure_; -} - -/** - * \brief Estimate the new exposure and gain values - * \param[inout] context The shared IPA Context - * \param[in] frameContext The FrameContext for this frame - * \param[in] yGain The gain calculated on the current brightness level - * \param[in] iqMeanGain The gain calculated based on the relative luminance target - */ -void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, - double yGain, double iqMeanGain) -{ - IPASessionConfiguration &configuration = context.configuration; - IPAActiveState &activeState = context.activeState; - - /* Get the effective exposure and gain applied on the sensor. */ - uint32_t exposure = frameContext.sensor.exposure; - double analogueGain = frameContext.sensor.gain; - - /* Use the highest of the two gain estimates. */ - double evGain = std::max(yGain, iqMeanGain); - - utils::Duration minShutterSpeed = configuration.sensor.minShutterSpeed; - utils::Duration maxShutterSpeed = std::min(configuration.sensor.maxShutterSpeed, - kMaxShutterSpeed); - - double minAnalogueGain = std::max(configuration.sensor.minAnalogueGain, - kMinAnalogueGain); - double maxAnalogueGain = configuration.sensor.maxAnalogueGain; + if (!frameContext.agc.autoExposureEnabled && frameContext.agc.autoExposureModeChange) { + context.activeState.agc.manual.exposure = activeAutoExposure; + frameContext.agc.exposure = activeAutoExposure; + } + if (!frameContext.agc.autoGainEnabled && frameContext.agc.autoGainModeChange) { + context.activeState.agc.manual.gain = activeAutoGain; + frameContext.agc.gain = activeAutoGain; + } - /* Consider within 1% of the target as correctly exposed. */ - if (utils::abs_diff(evGain, 1.0) < 0.01) + if (frame > 0 && !frameContext.agc.updateMetering) return; - /* extracted from Rpi::Agc::computeTargetExposure. */ - - /* Calculate the shutter time in seconds. */ - utils::Duration currentShutter = exposure * configuration.sensor.lineDuration; - - /* - * Update the exposure value for the next computation using the values - * of exposure and gain really used by the sensor. - */ - utils::Duration effectiveExposureValue = currentShutter * analogueGain; - - LOG(RkISP1Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain - << " Shutter speed " << currentShutter - << " Gain " << analogueGain - << " Needed ev gain " << evGain; - /* - * Calculate the current exposure value for the scene as the latest - * exposure value applied multiplied by the new estimated gain. + * Configure the AEC measurements. Set the window, measure + * continuously, and estimate Y as (R + G + B) x (85/256). */ - utils::Duration exposureValue = effectiveExposureValue * evGain; + auto aecConfig = params->block<BlockType::Aec>(); + aecConfig.setEnabled(true); - /* Clamp the exposure value to the min and max authorized. */ - utils::Duration maxTotalExposure = maxShutterSpeed * maxAnalogueGain; - exposureValue = std::min(exposureValue, maxTotalExposure); - LOG(RkISP1Agc, Debug) << "Target total exposure " << exposureValue - << ", maximum is " << maxTotalExposure; + aecConfig->meas_window = context.configuration.agc.measureWindow; + aecConfig->autostop = RKISP1_CIF_ISP_EXP_CTRL_AUTOSTOP_0; + aecConfig->mode = RKISP1_CIF_ISP_EXP_MEASURING_MODE_1; /* - * Divide the exposure value as new exposure and gain values. - * \todo estimate if we need to desaturate + * Configure the histogram measurement. Set the window, produce a + * luminance histogram, and set the weights and predivider. */ - exposureValue = filterExposure(exposureValue); - - /* - * Push the shutter time up to the maximum first, and only then - * increase the gain. - */ - utils::Duration shutterTime = std::clamp<utils::Duration>(exposureValue / minAnalogueGain, - minShutterSpeed, maxShutterSpeed); - double stepGain = std::clamp(exposureValue / shutterTime, - minAnalogueGain, maxAnalogueGain); - LOG(RkISP1Agc, Debug) << "Divided up shutter and gain are " - << shutterTime << " and " - << stepGain; + auto hstConfig = params->block<BlockType::Hst>(); + hstConfig.setEnabled(true); + + hstConfig->meas_window = context.configuration.agc.measureWindow; + hstConfig->mode = RKISP1_CIF_ISP_HISTOGRAM_MODE_Y_HISTOGRAM; + + Span<uint8_t> weights{ + hstConfig->hist_weight, + context.hw->numHistogramWeights + }; + std::vector<uint8_t> &modeWeights = meteringModes_.at(frameContext.agc.meteringMode); + std::copy(modeWeights.begin(), modeWeights.end(), weights.begin()); + + struct rkisp1_cif_isp_window window = hstConfig->meas_window; + Size windowSize = { window.h_size, window.v_size }; + hstConfig->histogram_predivider = + computeHistogramPredivider(windowSize, + static_cast<rkisp1_cif_isp_histogram_mode>(hstConfig->mode)); +} - /* Update the estimated exposure and gain. */ - activeState.agc.automatic.exposure = shutterTime / configuration.sensor.lineDuration; - activeState.agc.automatic.gain = stepGain; +void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, + ControlList &metadata) +{ + utils::Duration exposureTime = context.configuration.sensor.lineDuration + * frameContext.sensor.exposure; + metadata.set(controls::AnalogueGain, frameContext.sensor.gain); + metadata.set(controls::ExposureTime, exposureTime.get<std::micro>()); + metadata.set(controls::FrameDuration, frameContext.agc.frameDuration.get<std::micro>()); + metadata.set(controls::ExposureTimeMode, + frameContext.agc.autoExposureEnabled + ? controls::ExposureTimeModeAuto + : controls::ExposureTimeModeManual); + metadata.set(controls::AnalogueGainMode, + frameContext.agc.autoGainEnabled + ? controls::AnalogueGainModeAuto + : controls::AnalogueGainModeManual); + + metadata.set(controls::AeMeteringMode, frameContext.agc.meteringMode); + metadata.set(controls::AeExposureMode, frameContext.agc.exposureMode); + metadata.set(controls::AeConstraintMode, frameContext.agc.constraintMode); } /** * \brief Estimate the relative luminance of the frame with a given gain - * \param[in] ae The RkISP1 statistics and ISP results * \param[in] gain The gain to apply to the frame * * This function estimates the average relative luminance of the frame that @@ -332,8 +424,6 @@ void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, * YUV doesn't take into account the fact that the R, G and B components * contribute differently to the relative luminance. * - * \todo Have a dedicated YUV algorithm ? - * * The values are normalized to the [0.0, 1.0] range, where 1.0 corresponds to a * theoretical perfect reflector of 100% reference white. * @@ -342,46 +432,43 @@ void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext, * * \return The relative luminance */ -double Agc::estimateLuminance(const rkisp1_cif_isp_ae_stat *ae, - double gain) +double Agc::estimateLuminance(double gain) const { + ASSERT(expMeans_.size() == weights_.size()); double ySum = 0.0; + double wSum = 0.0; /* Sum the averages, saturated to 255. */ - for (unsigned int aeCell = 0; aeCell < numCells_; aeCell++) - ySum += std::min(ae->exp_mean[aeCell] * gain, 255.0); + for (unsigned i = 0; i < expMeans_.size(); i++) { + double w = weights_[i]; + ySum += std::min(expMeans_[i] * gain, 255.0) * w; + wSum += w; + } /* \todo Weight with the AWB gains */ - return ySum / numCells_ / 255; + return ySum / wSum / 255; } /** - * \brief Estimate the mean value of the top 2% of the histogram - * \param[in] hist The histogram statistics computed by the ImgU - * \return The mean value of the top 2% of the histogram + * \brief Process frame duration and compute vblank + * \param[in] context The shared IPA context + * \param[in] frameContext The current frame context + * \param[in] frameDuration The target frame duration + * + * Compute and populate vblank from the target frame duration. */ -double Agc::measureBrightness(const rkisp1_cif_isp_hist_stat *hist) const +void Agc::processFrameDuration(IPAContext &context, + IPAFrameContext &frameContext, + utils::Duration frameDuration) { - Histogram histogram{ Span<const uint32_t>(hist->hist_bins, numHistBins_) }; - /* Estimate the quantile mean of the top 2% of the histogram. */ - return histogram.interQuantileMean(0.98, 1.0); -} + IPACameraSensorInfo &sensorInfo = context.sensorInfo; + utils::Duration lineDuration = context.configuration.sensor.lineDuration; -void Agc::fillMetadata(IPAContext &context, IPAFrameContext &frameContext, - ControlList &metadata) -{ - utils::Duration exposureTime = context.configuration.sensor.lineDuration - * frameContext.sensor.exposure; - metadata.set(controls::AnalogueGain, frameContext.sensor.gain); - metadata.set(controls::ExposureTime, exposureTime.get<std::micro>()); + frameContext.agc.vblank = (frameDuration / lineDuration) - sensorInfo.outputSize.height; - /* \todo Use VBlank value calculated from each frame exposure. */ - uint32_t vTotal = context.configuration.sensor.size.height - + context.configuration.sensor.defVBlank; - utils::Duration frameDuration = context.configuration.sensor.lineDuration - * vTotal; - metadata.set(controls::FrameDuration, frameDuration.get<std::micro>()); + /* Update frame duration accounting for line length quantization. */ + frameContext.agc.frameDuration = (sensorInfo.outputSize.height + frameContext.agc.vblank) * lineDuration; } /** @@ -400,10 +487,20 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, ControlList &metadata) { if (!stats) { + processFrameDuration(context, frameContext, + frameContext.agc.minFrameDuration); + fillMetadata(context, frameContext, metadata); + return; + } + + if (!(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP)) { fillMetadata(context, frameContext, metadata); + LOG(RkISP1Agc, Error) << "AUTOEXP data is missing in statistics"; return; } + const utils::Duration &lineDuration = context.configuration.sensor.lineDuration; + /* * \todo Verify that the exposure and gain applied by the sensor for * this frame match what has been requested. This isn't a hard @@ -413,40 +510,77 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame, */ const rkisp1_cif_isp_stat *params = &stats->params; - ASSERT(stats->meas_type & RKISP1_CIF_ISP_STAT_AUTOEXP); - const rkisp1_cif_isp_ae_stat *ae = ¶ms->ae; - const rkisp1_cif_isp_hist_stat *hist = ¶ms->hist; - - double iqMean = measureBrightness(hist); - double iqMeanGain = kEvGainTarget * numHistBins_ / iqMean; + /* The lower 4 bits are fractional and meant to be discarded. */ + Histogram hist({ params->hist.hist_bins, context.hw->numHistogramBins }, + [](uint32_t x) { return x >> 4; }); + expMeans_ = { params->ae.exp_mean, context.hw->numAeCells }; + std::vector<uint8_t> &modeWeights = meteringModes_.at(frameContext.agc.meteringMode); + weights_ = { modeWeights.data(), modeWeights.size() }; /* - * Estimate the gain needed to achieve a relative luminance target. To - * account for non-linearity caused by saturation, the value needs to be - * estimated in an iterative process, as multiplying by a gain will not - * increase the relative luminance by the same factor if some image - * regions are saturated. + * Set the AGC limits using the fixed exposure time and/or gain in + * manual mode, or the sensor limits in auto mode. */ - double yGain = 1.0; - double yTarget = kRelativeLuminanceTarget; - - for (unsigned int i = 0; i < 8; i++) { - double yValue = estimateLuminance(ae, yGain); - double extra_gain = std::min(10.0, yTarget / (yValue + .001)); - - yGain *= extra_gain; - LOG(RkISP1Agc, Debug) << "Y value: " << yValue - << ", Y target: " << yTarget - << ", gives gain " << yGain; - if (extra_gain < 1.01) - break; + utils::Duration minExposureTime; + utils::Duration maxExposureTime; + double minAnalogueGain; + double maxAnalogueGain; + + if (frameContext.agc.autoExposureEnabled) { + minExposureTime = context.configuration.sensor.minExposureTime; + maxExposureTime = std::clamp(frameContext.agc.maxFrameDuration, + context.configuration.sensor.minExposureTime, + context.configuration.sensor.maxExposureTime); + } else { + minExposureTime = context.configuration.sensor.lineDuration + * frameContext.agc.exposure; + maxExposureTime = minExposureTime; } - computeExposure(context, frameContext, yGain, iqMeanGain); - frameCount_++; + if (frameContext.agc.autoGainEnabled) { + minAnalogueGain = context.configuration.sensor.minAnalogueGain; + maxAnalogueGain = context.configuration.sensor.maxAnalogueGain; + } else { + minAnalogueGain = frameContext.agc.gain; + maxAnalogueGain = frameContext.agc.gain; + } + + setLimits(minExposureTime, maxExposureTime, minAnalogueGain, maxAnalogueGain); + + /* + * The Agc algorithm needs to know the effective exposure value that was + * applied to the sensor when the statistics were collected. + */ + utils::Duration exposureTime = lineDuration * frameContext.sensor.exposure; + double analogueGain = frameContext.sensor.gain; + utils::Duration effectiveExposureValue = exposureTime * analogueGain; + + utils::Duration newExposureTime; + double aGain, dGain; + std::tie(newExposureTime, aGain, dGain) = + calculateNewEv(frameContext.agc.constraintMode, + frameContext.agc.exposureMode, + hist, effectiveExposureValue); + + LOG(RkISP1Agc, Debug) + << "Divided up exposure time, analogue gain and digital gain are " + << newExposureTime << ", " << aGain << " and " << dGain; + + IPAActiveState &activeState = context.activeState; + /* Update the estimated exposure and gain. */ + activeState.agc.automatic.exposure = newExposureTime / lineDuration; + activeState.agc.automatic.gain = aGain; + + /* + * Expand the target frame duration so that we do not run faster than + * the minimum frame duration when we have short exposures. + */ + processFrameDuration(context, frameContext, + std::max(frameContext.agc.minFrameDuration, newExposureTime)); fillMetadata(context, frameContext, metadata); + expMeans_ = {}; } REGISTER_IPA_ALGORITHM(Agc, "Agc") diff --git a/src/ipa/rkisp1/algorithms/agc.h b/src/ipa/rkisp1/algorithms/agc.h index 8a222637..7867eed9 100644 --- a/src/ipa/rkisp1/algorithms/agc.h +++ b/src/ipa/rkisp1/algorithms/agc.h @@ -2,29 +2,33 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * agc.h - RkISP1 AGC/AEC mean-based control algorithm + * RkISP1 AGC/AEC mean-based control algorithm */ #pragma once #include <linux/rkisp1-config.h> +#include <libcamera/base/span.h> #include <libcamera/base/utils.h> #include <libcamera/geometry.h> +#include "libipa/agc_mean_luminance.h" + #include "algorithm.h" namespace libcamera { namespace ipa::rkisp1::algorithms { -class Agc : public Algorithm +class Agc : public Algorithm, public AgcMeanLuminance { public: Agc(); ~Agc() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; void queueRequest(IPAContext &context, const uint32_t frame, @@ -32,27 +36,28 @@ public: const ControlList &controls) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) override; + RkISP1Params *params) override; void process(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const rkisp1_stat_buffer *stats, ControlList &metadata) override; private: - void computeExposure(IPAContext &Context, IPAFrameContext &frameContext, - double yGain, double iqMeanGain); - utils::Duration filterExposure(utils::Duration exposureValue); - double estimateLuminance(const rkisp1_cif_isp_ae_stat *ae, double gain); - double measureBrightness(const rkisp1_cif_isp_hist_stat *hist) const; + int parseMeteringModes(IPAContext &context, const YamlObject &tuningData); + uint8_t computeHistogramPredivider(const Size &size, + enum rkisp1_cif_isp_histogram_mode mode); + void fillMetadata(IPAContext &context, IPAFrameContext &frameContext, ControlList &metadata); + double estimateLuminance(double gain) const override; + void processFrameDuration(IPAContext &context, + IPAFrameContext &frameContext, + utils::Duration frameDuration); - uint64_t frameCount_; - - uint32_t numCells_; - uint32_t numHistBins_; + Span<const uint8_t> expMeans_; + Span<const uint8_t> weights_; - utils::Duration filteredExposure_; + std::map<int32_t, std::vector<uint8_t>> meteringModes_; }; } /* namespace ipa::rkisp1::algorithms */ diff --git a/src/ipa/rkisp1/algorithms/algorithm.h b/src/ipa/rkisp1/algorithms/algorithm.h index 9454c9a1..715cfcd8 100644 --- a/src/ipa/rkisp1/algorithms/algorithm.h +++ b/src/ipa/rkisp1/algorithms/algorithm.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2021, Ideas On Board * - * algorithm.h - RkISP1 control algorithm interface + * RkISP1 control algorithm interface */ #pragma once diff --git a/src/ipa/rkisp1/algorithms/awb.cpp b/src/ipa/rkisp1/algorithms/awb.cpp index 744f4a38..399fb51b 100644 --- a/src/ipa/rkisp1/algorithms/awb.cpp +++ b/src/ipa/rkisp1/algorithms/awb.cpp @@ -2,20 +2,24 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * awb.cpp - AWB control algorithm + * AWB control algorithm */ #include "awb.h" #include <algorithm> -#include <cmath> -#include <iomanip> +#include <ios> #include <libcamera/base/log.h> #include <libcamera/control_ids.h> + #include <libcamera/ipa/core_ipa_interface.h> +#include "libipa/awb_bayes.h" +#include "libipa/awb_grey.h" +#include "libipa/colours.h" + /** * \file awb.h */ @@ -31,27 +35,107 @@ namespace ipa::rkisp1::algorithms { LOG_DEFINE_CATEGORY(RkISP1Awb) +constexpr int32_t kMinColourTemperature = 2500; +constexpr int32_t kMaxColourTemperature = 10000; +constexpr int32_t kDefaultColourTemperature = 5000; + /* Minimum mean value below which AWB can't operate. */ constexpr double kMeanMinThreshold = 2.0; +class RkISP1AwbStats final : public AwbStats +{ +public: + RkISP1AwbStats(const RGB<double> &rgbMeans) + : rgbMeans_(rgbMeans) + { + rg_ = rgbMeans_.r() / rgbMeans_.g(); + bg_ = rgbMeans_.b() / rgbMeans_.g(); + } + + double computeColourError(const RGB<double> &gains) const override + { + /* + * Compute the sum of the squared colour error (non-greyness) as + * it appears in the log likelihood equation. + */ + double deltaR = gains.r() * rg_ - 1.0; + double deltaB = gains.b() * bg_ - 1.0; + double delta2 = deltaR * deltaR + deltaB * deltaB; + + return delta2; + } + + RGB<double> rgbMeans() const override + { + return rgbMeans_; + } + +private: + RGB<double> rgbMeans_; + double rg_; + double bg_; +}; + Awb::Awb() : rgbMode_(false) { } /** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Awb::init(IPAContext &context, const YamlObject &tuningData) +{ + auto &cmap = context.ctrlMap; + cmap[&controls::ColourTemperature] = ControlInfo(kMinColourTemperature, + kMaxColourTemperature, + kDefaultColourTemperature); + cmap[&controls::AwbEnable] = ControlInfo(false, true); + cmap[&controls::ColourGains] = ControlInfo(0.0f, 3.996f, 1.0f); + + if (!tuningData.contains("algorithm")) + LOG(RkISP1Awb, Info) << "No AWB algorithm specified." + << " Default to grey world"; + + auto mode = tuningData["algorithm"].get<std::string>("grey"); + if (mode == "grey") { + awbAlgo_ = std::make_unique<AwbGrey>(); + } else if (mode == "bayes") { + awbAlgo_ = std::make_unique<AwbBayes>(); + } else { + LOG(RkISP1Awb, Error) << "Unknown AWB algorithm: " << mode; + return -EINVAL; + } + LOG(RkISP1Awb, Debug) << "Using AWB algorithm: " << mode; + + int ret = awbAlgo_->init(tuningData); + if (ret) { + LOG(RkISP1Awb, Error) << "Failed to init AWB algorithm"; + return ret; + } + + const auto &src = awbAlgo_->controls(); + cmap.insert(src.begin(), src.end()); + + return 0; +} + +/** * \copydoc libcamera::ipa::Algorithm::configure */ int Awb::configure(IPAContext &context, const IPACameraSensorInfo &configInfo) { - context.activeState.awb.gains.manual.red = 1.0; - context.activeState.awb.gains.manual.blue = 1.0; - context.activeState.awb.gains.manual.green = 1.0; - context.activeState.awb.gains.automatic.red = 1.0; - context.activeState.awb.gains.automatic.blue = 1.0; - context.activeState.awb.gains.automatic.green = 1.0; + context.activeState.awb.manual.gains = RGB<double>{ 1.0 }; + auto gains = awbAlgo_->gainsFromColourTemperature(kDefaultColourTemperature); + if (gains) + context.activeState.awb.automatic.gains = *gains; + else + context.activeState.awb.automatic.gains = RGB<double>{ 1.0 }; + context.activeState.awb.autoEnabled = true; + context.activeState.awb.manual.temperatureK = kDefaultColourTemperature; + context.activeState.awb.automatic.temperatureK = kDefaultColourTemperature; /* * Define the measurement window for AWB as a centered rectangle @@ -85,64 +169,83 @@ void Awb::queueRequest(IPAContext &context, << (*awbEnable ? "Enabling" : "Disabling") << " AWB"; } - const auto &colourGains = controls.get(controls::ColourGains); - if (colourGains && !awb.autoEnabled) { - awb.gains.manual.red = (*colourGains)[0]; - awb.gains.manual.blue = (*colourGains)[1]; - - LOG(RkISP1Awb, Debug) - << "Set colour gains to red: " << awb.gains.manual.red - << ", blue: " << awb.gains.manual.blue; - } + awbAlgo_->handleControls(controls); frameContext.awb.autoEnabled = awb.autoEnabled; - if (!awb.autoEnabled) { - frameContext.awb.gains.red = awb.gains.manual.red; - frameContext.awb.gains.green = 1.0; - frameContext.awb.gains.blue = awb.gains.manual.blue; + if (awb.autoEnabled) + return; + + const auto &colourGains = controls.get(controls::ColourGains); + const auto &colourTemperature = controls.get(controls::ColourTemperature); + bool update = false; + if (colourGains) { + awb.manual.gains.r() = (*colourGains)[0]; + awb.manual.gains.b() = (*colourGains)[1]; + /* + * \todo Colour temperature reported in metadata is now + * incorrect, as we can't deduce the temperature from the gains. + * This will be fixed with the bayes AWB algorithm. + */ + update = true; + } else if (colourTemperature) { + awb.manual.temperatureK = *colourTemperature; + const auto &gains = awbAlgo_->gainsFromColourTemperature(*colourTemperature); + if (gains) { + awb.manual.gains.r() = gains->r(); + awb.manual.gains.b() = gains->b(); + update = true; + } } + + if (update) + LOG(RkISP1Awb, Debug) + << "Set colour gains to " << awb.manual.gains; + + frameContext.awb.gains = awb.manual.gains; + frameContext.awb.temperatureK = awb.manual.temperatureK; } /** * \copydoc libcamera::ipa::Algorithm::prepare */ void Awb::prepare(IPAContext &context, const uint32_t frame, - IPAFrameContext &frameContext, rkisp1_params_cfg *params) + IPAFrameContext &frameContext, RkISP1Params *params) { /* * This is the latest time we can read the active state. This is the * most up-to-date automatic values we can read. */ if (frameContext.awb.autoEnabled) { - frameContext.awb.gains.red = context.activeState.awb.gains.automatic.red; - frameContext.awb.gains.green = context.activeState.awb.gains.automatic.green; - frameContext.awb.gains.blue = context.activeState.awb.gains.automatic.blue; + const auto &awb = context.activeState.awb; + frameContext.awb.gains = awb.automatic.gains; + frameContext.awb.temperatureK = awb.automatic.temperatureK; } - params->others.awb_gain_config.gain_green_b = 256 * frameContext.awb.gains.green; - params->others.awb_gain_config.gain_blue = 256 * frameContext.awb.gains.blue; - params->others.awb_gain_config.gain_red = 256 * frameContext.awb.gains.red; - params->others.awb_gain_config.gain_green_r = 256 * frameContext.awb.gains.green; + auto gainConfig = params->block<BlockType::AwbGain>(); + gainConfig.setEnabled(true); - /* Update the gains. */ - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; + gainConfig->gain_green_b = std::clamp<int>(256 * frameContext.awb.gains.g(), 0, 0x3ff); + gainConfig->gain_blue = std::clamp<int>(256 * frameContext.awb.gains.b(), 0, 0x3ff); + gainConfig->gain_red = std::clamp<int>(256 * frameContext.awb.gains.r(), 0, 0x3ff); + gainConfig->gain_green_r = std::clamp<int>(256 * frameContext.awb.gains.g(), 0, 0x3ff); /* If we have already set the AWB measurement parameters, return. */ if (frame > 0) return; - rkisp1_cif_isp_awb_meas_config &awb_config = params->meas.awb_meas_config; + auto awbConfig = params->block<BlockType::Awb>(); + awbConfig.setEnabled(true); /* Configure the measure window for AWB. */ - awb_config.awb_wnd = context.configuration.awb.measureWindow; + awbConfig->awb_wnd = context.configuration.awb.measureWindow; /* Number of frames to use to estimate the means (0 means 1 frame). */ - awb_config.frames = 0; + awbConfig->frames = 0; /* Select RGB or YCbCr means measurement. */ if (rgbMode_) { - awb_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_RGB; + awbConfig->awb_mode = RKISP1_CIF_ISP_AWB_MODE_RGB; /* * For RGB-based measurements, pixels are selected with maximum @@ -150,19 +253,19 @@ void Awb::prepare(IPAContext &context, const uint32_t frame, * awb_ref_cr, awb_min_y and awb_ref_cb respectively. The other * values are not used, set them to 0. */ - awb_config.awb_ref_cr = 250; - awb_config.min_y = 250; - awb_config.awb_ref_cb = 250; + awbConfig->awb_ref_cr = 250; + awbConfig->min_y = 250; + awbConfig->awb_ref_cb = 250; - awb_config.max_y = 0; - awb_config.min_c = 0; - awb_config.max_csum = 0; + awbConfig->max_y = 0; + awbConfig->min_c = 0; + awbConfig->max_csum = 0; } else { - awb_config.awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR; + awbConfig->awb_mode = RKISP1_CIF_ISP_AWB_MODE_YCBCR; /* Set the reference Cr and Cb (AWB target) to white. */ - awb_config.awb_ref_cb = 128; - awb_config.awb_ref_cr = 128; + awbConfig->awb_ref_cb = 128; + awbConfig->awb_ref_cr = 128; /* * Filter out pixels based on luminance and chrominance values. @@ -170,36 +273,11 @@ void Awb::prepare(IPAContext &context, const uint32_t frame, * range, while the acceptable chroma values are specified with * a minimum of 16 and a maximum Cb+Cr sum of 250. */ - awb_config.min_y = 16; - awb_config.max_y = 250; - awb_config.min_c = 16; - awb_config.max_csum = 250; + awbConfig->min_y = 16; + awbConfig->max_y = 250; + awbConfig->min_c = 16; + awbConfig->max_csum = 250; } - - /* Enable the AWB gains. */ - params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; - params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB_GAIN; - - /* Update the AWB measurement parameters and enable the AWB module. */ - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_AWB; - params->module_en_update |= RKISP1_CIF_ISP_MODULE_AWB; - params->module_ens |= RKISP1_CIF_ISP_MODULE_AWB; -} - -uint32_t Awb::estimateCCT(double red, double green, double blue) -{ - /* Convert the RGB values to CIE tristimulus values (XYZ) */ - double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue); - double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue); - double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue); - - /* Calculate the normalized chromaticity values */ - double x = X / (X + Y + Z); - double y = Y / (X + Y + Z); - - /* Calculate CCT */ - double n = (x - 0.3320) / (0.1858 - y); - return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33; } /** @@ -211,41 +289,111 @@ void Awb::process(IPAContext &context, const rkisp1_stat_buffer *stats, ControlList &metadata) { + IPAActiveState &activeState = context.activeState; + + metadata.set(controls::AwbEnable, frameContext.awb.autoEnabled); + metadata.set(controls::ColourGains, { + static_cast<float>(frameContext.awb.gains.r()), + static_cast<float>(frameContext.awb.gains.b()) + }); + metadata.set(controls::ColourTemperature, frameContext.awb.temperatureK); + + if (!stats || !(stats->meas_type & RKISP1_CIF_ISP_STAT_AWB)) { + LOG(RkISP1Awb, Error) << "AWB data is missing in statistics"; + return; + } + const rkisp1_cif_isp_stat *params = &stats->params; const rkisp1_cif_isp_awb_stat *awb = ¶ms->awb; - IPAActiveState &activeState = context.activeState; - double greenMean; - double redMean; - double blueMean; + + if (awb->awb_mean[0].cnt == 0) { + LOG(RkISP1Awb, Debug) << "AWB statistics are empty"; + return; + } + + RGB<double> rgbMeans = calculateRgbMeans(frameContext, awb); + + /* + * If the means are too small we don't have enough information to + * meaningfully calculate gains. Freeze the algorithm in that case. + */ + if (rgbMeans.r() < kMeanMinThreshold && rgbMeans.g() < kMeanMinThreshold && + rgbMeans.b() < kMeanMinThreshold) + return; + + RkISP1AwbStats awbStats{ rgbMeans }; + AwbResult awbResult = awbAlgo_->calculateAwb(awbStats, frameContext.lux.lux); + + /* + * Clamp the gain values to the hardware, which expresses gains as Q2.8 + * unsigned integer values. Set the minimum just above zero to avoid + * divisions by zero when computing the raw means in subsequent + * iterations. + */ + awbResult.gains = awbResult.gains.max(1.0 / 256).min(1023.0 / 256); + + /* Filter the values to avoid oscillations. */ + double speed = 0.2; + double ct = awbResult.colourTemperature; + ct = ct * speed + activeState.awb.automatic.temperatureK * (1 - speed); + awbResult.gains = awbResult.gains * speed + + activeState.awb.automatic.gains * (1 - speed); + + activeState.awb.automatic.temperatureK = static_cast<unsigned int>(ct); + activeState.awb.automatic.gains = awbResult.gains; + + LOG(RkISP1Awb, Debug) + << std::showpoint + << "Means " << rgbMeans << ", gains " + << activeState.awb.automatic.gains << ", temp " + << activeState.awb.automatic.temperatureK << "K"; +} + +RGB<double> Awb::calculateRgbMeans(const IPAFrameContext &frameContext, const rkisp1_cif_isp_awb_stat *awb) const +{ + Vector<double, 3> rgbMeans; if (rgbMode_) { - greenMean = awb->awb_mean[0].mean_y_or_g; - redMean = awb->awb_mean[0].mean_cr_or_r; - blueMean = awb->awb_mean[0].mean_cb_or_b; + rgbMeans = {{ + static_cast<double>(awb->awb_mean[0].mean_cr_or_r), + static_cast<double>(awb->awb_mean[0].mean_y_or_g), + static_cast<double>(awb->awb_mean[0].mean_cb_or_b) + }}; } else { /* Get the YCbCr mean values */ - double yMean = awb->awb_mean[0].mean_y_or_g; - double cbMean = awb->awb_mean[0].mean_cb_or_b; - double crMean = awb->awb_mean[0].mean_cr_or_r; + Vector<double, 3> yuvMeans({ + static_cast<double>(awb->awb_mean[0].mean_y_or_g), + static_cast<double>(awb->awb_mean[0].mean_cb_or_b), + static_cast<double>(awb->awb_mean[0].mean_cr_or_r) + }); /* - * Convert from YCbCr to RGB. - * The hardware uses the following formulas: - * Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B + * Convert from YCbCr to RGB. The hardware uses the following + * formulas: + * + * Y = 16 + 0.2500 R + 0.5000 G + 0.1094 B * Cb = 128 - 0.1406 R - 0.2969 G + 0.4375 B * Cr = 128 + 0.4375 R - 0.3750 G - 0.0625 B * - * The inverse matrix is thus: + * This seems to be based on limited range BT.601 with Q1.6 + * precision. + * + * The inverse matrix is: + * * [[1,1636, -0,0623, 1,6008] * [1,1636, -0,4045, -0,7949] * [1,1636, 1,9912, -0,0250]] */ - yMean -= 16; - cbMean -= 128; - crMean -= 128; - redMean = 1.1636 * yMean - 0.0623 * cbMean + 1.6008 * crMean; - greenMean = 1.1636 * yMean - 0.4045 * cbMean - 0.7949 * crMean; - blueMean = 1.1636 * yMean + 1.9912 * cbMean - 0.0250 * crMean; + static const Matrix<double, 3, 3> yuv2rgbMatrix({ + 1.1636, -0.0623, 1.6008, + 1.1636, -0.4045, -0.7949, + 1.1636, 1.9912, -0.0250 + }); + static const Vector<double, 3> yuv2rgbOffset({ + 16, 128, 128 + }); + + rgbMeans = yuv2rgbMatrix * (yuvMeans - yuv2rgbOffset); /* * Due to hardware rounding errors in the YCbCr means, the @@ -253,73 +401,23 @@ void Awb::process(IPAContext &context, * negative gains, messing up calculation. Prevent this by * clamping the means to positive values. */ - redMean = std::max(redMean, 0.0); - greenMean = std::max(greenMean, 0.0); - blueMean = std::max(blueMean, 0.0); + rgbMeans = rgbMeans.max(0.0); } /* - * The ISP computes the AWB means after applying the colour gains, - * divide by the gains that were used to get the raw means from the - * sensor. - */ - redMean /= frameContext.awb.gains.red; - greenMean /= frameContext.awb.gains.green; - blueMean /= frameContext.awb.gains.blue; - - /* - * If the means are too small we don't have enough information to - * meaningfully calculate gains. Freeze the algorithm in that case. - */ - if (redMean < kMeanMinThreshold && greenMean < kMeanMinThreshold && - blueMean < kMeanMinThreshold) { - frameContext.awb.temperatureK = activeState.awb.temperatureK; - return; - } - - activeState.awb.temperatureK = estimateCCT(redMean, greenMean, blueMean); - - /* - * Estimate the red and blue gains to apply in a grey world. The green - * gain is hardcoded to 1.0. Avoid divisions by zero by clamping the - * divisor to a minimum value of 1.0. + * The ISP computes the AWB means after applying the CCM. Apply the + * inverse as we want to get the raw means before the colour gains. */ - double redGain = greenMean / std::max(redMean, 1.0); - double blueGain = greenMean / std::max(blueMean, 1.0); + rgbMeans = frameContext.ccm.ccm.inverse() * rgbMeans; /* - * Clamp the gain values to the hardware, which expresses gains as Q2.8 - * unsigned integer values. Set the minimum just above zero to avoid - * divisions by zero when computing the raw means in subsequent - * iterations. + * The ISP computes the AWB means after applying the colour gains, + * divide by the gains that were used to get the raw means from the + * sensor. Apply a minimum value to avoid divisions by near-zero. */ - redGain = std::clamp(redGain, 1.0 / 256, 1023.0 / 256); - blueGain = std::clamp(blueGain, 1.0 / 256, 1023.0 / 256); - - /* Filter the values to avoid oscillations. */ - double speed = 0.2; - redGain = speed * redGain + (1 - speed) * activeState.awb.gains.automatic.red; - blueGain = speed * blueGain + (1 - speed) * activeState.awb.gains.automatic.blue; - - activeState.awb.gains.automatic.red = redGain; - activeState.awb.gains.automatic.blue = blueGain; - activeState.awb.gains.automatic.green = 1.0; - - frameContext.awb.temperatureK = activeState.awb.temperatureK; - - metadata.set(controls::AwbEnable, frameContext.awb.autoEnabled); - metadata.set(controls::ColourGains, { - static_cast<float>(frameContext.awb.gains.red), - static_cast<float>(frameContext.awb.gains.blue) - }); - metadata.set(controls::ColourTemperature, frameContext.awb.temperatureK); + rgbMeans /= frameContext.awb.gains.max(0.01); - LOG(RkISP1Awb, Debug) << std::showpoint - << "Means [" << redMean << ", " << greenMean << ", " << blueMean - << "], gains [" << activeState.awb.gains.automatic.red << ", " - << activeState.awb.gains.automatic.green << ", " - << activeState.awb.gains.automatic.blue << "], temp " - << frameContext.awb.temperatureK << "K"; + return rgbMeans; } REGISTER_IPA_ALGORITHM(Awb, "Awb") diff --git a/src/ipa/rkisp1/algorithms/awb.h b/src/ipa/rkisp1/algorithms/awb.h index 9d45a442..02651cc7 100644 --- a/src/ipa/rkisp1/algorithms/awb.h +++ b/src/ipa/rkisp1/algorithms/awb.h @@ -2,11 +2,16 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * awb.h - AWB control algorithm + * AWB control algorithm */ #pragma once +#include "libcamera/internal/vector.h" + +#include "libipa/awb.h" +#include "libipa/interpolator.h" + #include "algorithm.h" namespace libcamera { @@ -19,20 +24,24 @@ public: Awb(); ~Awb() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; void queueRequest(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const ControlList &controls) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) override; + RkISP1Params *params) override; void process(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const rkisp1_stat_buffer *stats, ControlList &metadata) override; private: - uint32_t estimateCCT(double red, double green, double blue); + RGB<double> calculateRgbMeans(const IPAFrameContext &frameContext, + const rkisp1_cif_isp_awb_stat *awb) const; + + std::unique_ptr<AwbAlgorithm> awbAlgo_; bool rgbMode_; }; diff --git a/src/ipa/rkisp1/algorithms/blc.cpp b/src/ipa/rkisp1/algorithms/blc.cpp index 15324fb1..98cb7145 100644 --- a/src/ipa/rkisp1/algorithms/blc.cpp +++ b/src/ipa/rkisp1/algorithms/blc.cpp @@ -2,13 +2,17 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * blc.cpp - RkISP1 Black Level Correction control + * RkISP1 Black Level Correction control */ #include "blc.h" +#include <linux/videodev2.h> + #include <libcamera/base/log.h> +#include <libcamera/control_ids.h> + #include "libcamera/internal/yaml_parser.h" /** @@ -36,22 +40,62 @@ namespace ipa::rkisp1::algorithms { LOG_DEFINE_CATEGORY(RkISP1Blc) BlackLevelCorrection::BlackLevelCorrection() - : tuningParameters_(false) { + /* + * This is a bit of a hack. In raw mode no black level correction + * happens. This flag is used to ensure the metadata gets populated with + * the black level which is needed to capture proper raw images for + * tuning. + */ + supportsRaw_ = true; } /** * \copydoc libcamera::ipa::Algorithm::init */ -int BlackLevelCorrection::init([[maybe_unused]] IPAContext &context, - const YamlObject &tuningData) +int BlackLevelCorrection::init(IPAContext &context, const YamlObject &tuningData) { - blackLevelRed_ = tuningData["R"].get<int16_t>(256); - blackLevelGreenR_ = tuningData["Gr"].get<int16_t>(256); - blackLevelGreenB_ = tuningData["Gb"].get<int16_t>(256); - blackLevelBlue_ = tuningData["B"].get<int16_t>(256); - - tuningParameters_ = true; + std::optional<int16_t> levelRed = tuningData["R"].get<int16_t>(); + std::optional<int16_t> levelGreenR = tuningData["Gr"].get<int16_t>(); + std::optional<int16_t> levelGreenB = tuningData["Gb"].get<int16_t>(); + std::optional<int16_t> levelBlue = tuningData["B"].get<int16_t>(); + bool tuningHasLevels = levelRed && levelGreenR && levelGreenB && levelBlue; + + auto blackLevel = context.camHelper->blackLevel(); + if (!blackLevel) { + /* + * Not all camera sensor helpers have been updated with black + * levels. Print a warning and fall back to the levels from the + * tuning data to preserve backward compatibility. This should + * be removed once all helpers provide the data. + */ + LOG(RkISP1Blc, Warning) + << "No black levels provided by camera sensor helper" + << ", please fix"; + + blackLevelRed_ = levelRed.value_or(4096); + blackLevelGreenR_ = levelGreenR.value_or(4096); + blackLevelGreenB_ = levelGreenB.value_or(4096); + blackLevelBlue_ = levelBlue.value_or(4096); + } else if (tuningHasLevels) { + /* + * If black levels are provided in the tuning file, use them to + * avoid breaking existing camera tuning. This is deprecated and + * will be removed. + */ + LOG(RkISP1Blc, Warning) + << "Deprecated: black levels overwritten by tuning file"; + + blackLevelRed_ = *levelRed; + blackLevelGreenR_ = *levelGreenR; + blackLevelGreenB_ = *levelGreenB; + blackLevelBlue_ = *levelBlue; + } else { + blackLevelRed_ = *blackLevel; + blackLevelGreenR_ = *blackLevel; + blackLevelGreenB_ = *blackLevel; + blackLevelBlue_ = *blackLevel; + } LOG(RkISP1Blc, Debug) << "Black levels: red " << blackLevelRed_ @@ -62,29 +106,80 @@ int BlackLevelCorrection::init([[maybe_unused]] IPAContext &context, return 0; } +int BlackLevelCorrection::configure(IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + /* + * BLC on ISP versions that include the companding block requires usage + * of the extensible parameters format. + */ + supported_ = context.configuration.paramFormat == V4L2_META_FMT_RK_ISP1_EXT_PARAMS || + !context.hw->compand; + + if (!supported_) + LOG(RkISP1Blc, Warning) + << "BLC in companding block requires extensible parameters"; + + return 0; +} + /** * \copydoc libcamera::ipa::Algorithm::prepare */ -void BlackLevelCorrection::prepare([[maybe_unused]] IPAContext &context, +void BlackLevelCorrection::prepare(IPAContext &context, const uint32_t frame, [[maybe_unused]] IPAFrameContext &frameContext, - rkisp1_params_cfg *params) + RkISP1Params *params) { + if (context.configuration.raw) + return; + if (frame > 0) return; - if (!tuningParameters_) + if (!supported_) return; - params->others.bls_config.enable_auto = 0; - params->others.bls_config.fixed_val.r = blackLevelRed_; - params->others.bls_config.fixed_val.gr = blackLevelGreenR_; - params->others.bls_config.fixed_val.gb = blackLevelGreenB_; - params->others.bls_config.fixed_val.b = blackLevelBlue_; + if (context.hw->compand) { + auto config = params->block<BlockType::CompandBls>(); + config.setEnabled(true); + + /* + * Scale up to the 20-bit black levels used by the companding + * block. + */ + config->r = blackLevelRed_ << 4; + config->gr = blackLevelGreenR_ << 4; + config->gb = blackLevelGreenB_ << 4; + config->b = blackLevelBlue_ << 4; + } else { + auto config = params->block<BlockType::Bls>(); + config.setEnabled(true); + + config->enable_auto = 0; + + /* Scale down to the 12-bit black levels used by the BLS block. */ + config->fixed_val.r = blackLevelRed_ >> 4; + config->fixed_val.gr = blackLevelGreenR_ >> 4; + config->fixed_val.gb = blackLevelGreenB_ >> 4; + config->fixed_val.b = blackLevelBlue_ >> 4; + } +} - params->module_en_update |= RKISP1_CIF_ISP_MODULE_BLS; - params->module_ens |= RKISP1_CIF_ISP_MODULE_BLS; - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_BLS; +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void BlackLevelCorrection::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const rkisp1_stat_buffer *stats, + ControlList &metadata) +{ + metadata.set(controls::SensorBlackLevels, + { static_cast<int32_t>(blackLevelRed_), + static_cast<int32_t>(blackLevelGreenR_), + static_cast<int32_t>(blackLevelGreenB_), + static_cast<int32_t>(blackLevelBlue_) }); } REGISTER_IPA_ALGORITHM(BlackLevelCorrection, "BlackLevelCorrection") diff --git a/src/ipa/rkisp1/algorithms/blc.h b/src/ipa/rkisp1/algorithms/blc.h index 0b1a2d43..f797ae44 100644 --- a/src/ipa/rkisp1/algorithms/blc.h +++ b/src/ipa/rkisp1/algorithms/blc.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * blc.h - RkISP1 Black Level Correction control + * RkISP1 Black Level Correction control */ #pragma once @@ -20,12 +20,19 @@ public: ~BlackLevelCorrection() = default; int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) override; + RkISP1Params *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats, + ControlList &metadata) override; private: - bool tuningParameters_; + bool supported_; + int16_t blackLevelRed_; int16_t blackLevelGreenR_; int16_t blackLevelGreenB_; diff --git a/src/ipa/rkisp1/algorithms/ccm.cpp b/src/ipa/rkisp1/algorithms/ccm.cpp new file mode 100644 index 00000000..de2b6fe7 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/ccm.cpp @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * RkISP1 Color Correction Matrix control algorithm + */ + +#include "ccm.h" + +#include <map> + +#include <libcamera/base/log.h> +#include <libcamera/base/utils.h> + +#include <libcamera/control_ids.h> + +#include <libcamera/ipa/core_ipa_interface.h> + +#include "libcamera/internal/yaml_parser.h" + +#include "libipa/fixedpoint.h" +#include "libipa/interpolator.h" + +/** + * \file ccm.h + */ + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +/** + * \class Ccm + * \brief A color correction matrix algorithm + */ + +LOG_DEFINE_CATEGORY(RkISP1Ccm) + +constexpr Matrix<float, 3, 3> kIdentity3x3 = Matrix<float, 3, 3>::identity(); + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Ccm::init([[maybe_unused]] IPAContext &context, const YamlObject &tuningData) +{ + auto &cmap = context.ctrlMap; + cmap[&controls::ColourCorrectionMatrix] = ControlInfo( + ControlValue(-8.0f), + ControlValue(7.993f), + ControlValue(kIdentity3x3.data())); + + int ret = ccm_.readYaml(tuningData["ccms"], "ct", "ccm"); + if (ret < 0) { + LOG(RkISP1Ccm, Warning) + << "Failed to parse 'ccm' " + << "parameter from tuning file; falling back to unit matrix"; + ccm_.setData({ { 0, kIdentity3x3 } }); + } + + ret = offsets_.readYaml(tuningData["ccms"], "ct", "offsets"); + if (ret < 0) { + LOG(RkISP1Ccm, Warning) + << "Failed to parse 'offsets' " + << "parameter from tuning file; falling back to zero offsets"; + + offsets_.setData({ { 0, Matrix<int16_t, 3, 1>({ 0, 0, 0 }) } }); + } + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::configure + */ +int Ccm::configure(IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + auto &as = context.activeState; + as.ccm.manual = kIdentity3x3; + as.ccm.automatic = ccm_.getInterpolated(as.awb.automatic.temperatureK); + return 0; +} + +void Ccm::queueRequest(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const ControlList &controls) +{ + /* Nothing to do here, the ccm will be calculated in prepare() */ + if (frameContext.awb.autoEnabled) + return; + + auto &ccm = context.activeState.ccm; + + const auto &colourTemperature = controls.get(controls::ColourTemperature); + const auto &ccmMatrix = controls.get(controls::ColourCorrectionMatrix); + if (ccmMatrix) { + ccm.manual = Matrix<float, 3, 3>(*ccmMatrix); + LOG(RkISP1Ccm, Debug) + << "Setting manual CCM from CCM control to " << ccm.manual; + } else if (colourTemperature) { + ccm.manual = ccm_.getInterpolated(*colourTemperature); + LOG(RkISP1Ccm, Debug) + << "Setting manual CCM from CT control to " << ccm.manual; + } + + frameContext.ccm.ccm = ccm.manual; +} + +void Ccm::setParameters(struct rkisp1_cif_isp_ctk_config &config, + const Matrix<float, 3, 3> &matrix, + const Matrix<int16_t, 3, 1> &offsets) +{ + /* + * 4 bit integer and 7 bit fractional, ranging from -8 (0x400) to + * +7.9921875 (0x3ff) + */ + for (unsigned int i = 0; i < 3; i++) { + for (unsigned int j = 0; j < 3; j++) + config.coeff[i][j] = + floatingToFixedPoint<4, 7, uint16_t, double>(matrix[i][j]); + } + + for (unsigned int i = 0; i < 3; i++) + config.ct_offset[i] = offsets[i][0] & 0xfff; + + LOG(RkISP1Ccm, Debug) << "Setting matrix " << matrix; + LOG(RkISP1Ccm, Debug) << "Setting offsets " << offsets; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void Ccm::prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, RkISP1Params *params) +{ + if (!frameContext.awb.autoEnabled) { + auto config = params->block<BlockType::Ctk>(); + config.setEnabled(true); + setParameters(*config, frameContext.ccm.ccm, Matrix<int16_t, 3, 1>()); + return; + } + + uint32_t ct = frameContext.awb.temperatureK; + if (frame > 0 && ct == ct_) { + frameContext.ccm.ccm = context.activeState.ccm.automatic; + return; + } + + ct_ = ct; + Matrix<float, 3, 3> ccm = ccm_.getInterpolated(ct); + Matrix<int16_t, 3, 1> offsets = offsets_.getInterpolated(ct); + + context.activeState.ccm.automatic = ccm; + frameContext.ccm.ccm = ccm; + + auto config = params->block<BlockType::Ctk>(); + config.setEnabled(true); + setParameters(*config, ccm, offsets); +} + +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void Ccm::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + [[maybe_unused]] const rkisp1_stat_buffer *stats, + ControlList &metadata) +{ + metadata.set(controls::ColourCorrectionMatrix, frameContext.ccm.ccm.data()); +} + +REGISTER_IPA_ALGORITHM(Ccm, "Ccm") + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/ccm.h b/src/ipa/rkisp1/algorithms/ccm.h new file mode 100644 index 00000000..c301e6e5 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/ccm.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * RkISP1 Color Correction Matrix control algorithm + */ + +#pragma once + +#include <linux/rkisp1-config.h> + +#include "libcamera/internal/matrix.h" + +#include "libipa/interpolator.h" + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +class Ccm : public Algorithm +{ +public: + Ccm() {} + ~Ccm() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) override; + void queueRequest(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + const ControlList &controls) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + RkISP1Params *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats, + ControlList &metadata) override; + +private: + void parseYaml(const YamlObject &tuningData); + void setParameters(struct rkisp1_cif_isp_ctk_config &config, + const Matrix<float, 3, 3> &matrix, + const Matrix<int16_t, 3, 1> &offsets); + + unsigned int ct_; + Interpolator<Matrix<float, 3, 3>> ccm_; + Interpolator<Matrix<int16_t, 3, 1>> offsets_; +}; + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/cproc.cpp b/src/ipa/rkisp1/algorithms/cproc.cpp index eaa56c37..d1fff699 100644 --- a/src/ipa/rkisp1/algorithms/cproc.cpp +++ b/src/ipa/rkisp1/algorithms/cproc.cpp @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * cproc.cpp - RkISP1 Color Processing control + * RkISP1 Color Processing control */ #include "cproc.h" @@ -33,20 +33,71 @@ namespace ipa::rkisp1::algorithms { LOG_DEFINE_CATEGORY(RkISP1CProc) +namespace { + +constexpr float kDefaultBrightness = 0.0f; +constexpr float kDefaultContrast = 1.0f; +constexpr float kDefaultSaturation = 1.0f; + +int convertBrightness(const float v) +{ + return std::clamp<int>(std::lround(v * 128), -128, 127); +} + +int convertContrastOrSaturation(const float v) +{ + return std::clamp<int>(std::lround(v * 128), 0, 255); +} + +} /* namespace */ + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int ColorProcessing::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + auto &cmap = context.ctrlMap; + + cmap[&controls::Brightness] = ControlInfo(-1.0f, 0.993f, kDefaultBrightness); + cmap[&controls::Contrast] = ControlInfo(0.0f, 1.993f, kDefaultContrast); + cmap[&controls::Saturation] = ControlInfo(0.0f, 1.993f, kDefaultSaturation); + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::configure + */ +int ColorProcessing::configure(IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + auto &cproc = context.activeState.cproc; + + cproc.brightness = convertBrightness(kDefaultBrightness); + cproc.contrast = convertContrastOrSaturation(kDefaultContrast); + cproc.saturation = convertContrastOrSaturation(kDefaultSaturation); + + return 0; +} + /** * \copydoc libcamera::ipa::Algorithm::queueRequest */ void ColorProcessing::queueRequest(IPAContext &context, - [[maybe_unused]] const uint32_t frame, + const uint32_t frame, IPAFrameContext &frameContext, const ControlList &controls) { auto &cproc = context.activeState.cproc; bool update = false; + if (frame == 0) + update = true; + const auto &brightness = controls.get(controls::Brightness); if (brightness) { - int value = std::clamp<int>(std::lround(*brightness * 128), -128, 127); + int value = convertBrightness(*brightness); if (cproc.brightness != value) { cproc.brightness = value; update = true; @@ -57,7 +108,7 @@ void ColorProcessing::queueRequest(IPAContext &context, const auto &contrast = controls.get(controls::Contrast); if (contrast) { - int value = std::clamp<int>(std::lround(*contrast * 128), 0, 255); + int value = convertContrastOrSaturation(*contrast); if (cproc.contrast != value) { cproc.contrast = value; update = true; @@ -68,7 +119,7 @@ void ColorProcessing::queueRequest(IPAContext &context, const auto saturation = controls.get(controls::Saturation); if (saturation) { - int value = std::clamp<int>(std::lround(*saturation * 128), 0, 255); + int value = convertContrastOrSaturation(*saturation); if (cproc.saturation != value) { cproc.saturation = value; update = true; @@ -89,19 +140,17 @@ void ColorProcessing::queueRequest(IPAContext &context, void ColorProcessing::prepare([[maybe_unused]] IPAContext &context, [[maybe_unused]] const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) + RkISP1Params *params) { /* Check if the algorithm configuration has been updated. */ if (!frameContext.cproc.update) return; - params->others.cproc_config.brightness = frameContext.cproc.brightness; - params->others.cproc_config.contrast = frameContext.cproc.contrast; - params->others.cproc_config.sat = frameContext.cproc.saturation; - - params->module_en_update |= RKISP1_CIF_ISP_MODULE_CPROC; - params->module_ens |= RKISP1_CIF_ISP_MODULE_CPROC; - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_CPROC; + auto config = params->block<BlockType::Cproc>(); + config.setEnabled(true); + config->brightness = frameContext.cproc.brightness; + config->contrast = frameContext.cproc.contrast; + config->sat = frameContext.cproc.saturation; } REGISTER_IPA_ALGORITHM(ColorProcessing, "ColorProcessing") diff --git a/src/ipa/rkisp1/algorithms/cproc.h b/src/ipa/rkisp1/algorithms/cproc.h index ba6e901a..fd38fd17 100644 --- a/src/ipa/rkisp1/algorithms/cproc.h +++ b/src/ipa/rkisp1/algorithms/cproc.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * cproc.h - RkISP1 Color Processing control + * RkISP1 Color Processing control */ #pragma once @@ -21,12 +21,15 @@ public: ColorProcessing() = default; ~ColorProcessing() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) override; void queueRequest(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const ControlList &controls) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) override; + RkISP1Params *params) override; }; } /* namespace ipa::rkisp1::algorithms */ diff --git a/src/ipa/rkisp1/algorithms/dpcc.cpp b/src/ipa/rkisp1/algorithms/dpcc.cpp index 80a1b734..78946281 100644 --- a/src/ipa/rkisp1/algorithms/dpcc.cpp +++ b/src/ipa/rkisp1/algorithms/dpcc.cpp @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * dpcc.cpp - RkISP1 Defect Pixel Cluster Correction control + * RkISP1 Defect Pixel Cluster Correction control */ #include "dpcc.h" @@ -232,16 +232,14 @@ int DefectPixelClusterCorrection::init([[maybe_unused]] IPAContext &context, void DefectPixelClusterCorrection::prepare([[maybe_unused]] IPAContext &context, const uint32_t frame, [[maybe_unused]] IPAFrameContext &frameContext, - rkisp1_params_cfg *params) + RkISP1Params *params) { if (frame > 0) return; - params->others.dpcc_config = config_; - - params->module_en_update |= RKISP1_CIF_ISP_MODULE_DPCC; - params->module_ens |= RKISP1_CIF_ISP_MODULE_DPCC; - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_DPCC; + auto config = params->block<BlockType::Dpcc>(); + config.setEnabled(true); + *config = config_; } REGISTER_IPA_ALGORITHM(DefectPixelClusterCorrection, "DefectPixelClusterCorrection") diff --git a/src/ipa/rkisp1/algorithms/dpcc.h b/src/ipa/rkisp1/algorithms/dpcc.h index b1fac7d1..b77766c3 100644 --- a/src/ipa/rkisp1/algorithms/dpcc.h +++ b/src/ipa/rkisp1/algorithms/dpcc.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * dpcc.h - RkISP1 Defect Pixel Cluster Correction control + * RkISP1 Defect Pixel Cluster Correction control */ #pragma once @@ -22,7 +22,7 @@ public: int init(IPAContext &context, const YamlObject &tuningData) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) override; + RkISP1Params *params) override; private: rkisp1_cif_isp_dpcc_config config_; diff --git a/src/ipa/rkisp1/algorithms/dpf.cpp b/src/ipa/rkisp1/algorithms/dpf.cpp index 5bd7e59f..cb6095da 100644 --- a/src/ipa/rkisp1/algorithms/dpf.cpp +++ b/src/ipa/rkisp1/algorithms/dpf.cpp @@ -2,12 +2,14 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * dpf.cpp - RkISP1 Denoise Pre-Filter control + * RkISP1 Denoise Pre-Filter control */ #include "dpf.h" -#include <cmath> +#include <algorithm> +#include <string> +#include <vector> #include <libcamera/base/log.h> @@ -215,15 +217,21 @@ void Dpf::queueRequest(IPAContext &context, * \copydoc libcamera::ipa::Algorithm::prepare */ void Dpf::prepare(IPAContext &context, const uint32_t frame, - IPAFrameContext &frameContext, rkisp1_params_cfg *params) + IPAFrameContext &frameContext, RkISP1Params *params) { - if (frame == 0) { - params->others.dpf_config = config_; - params->others.dpf_strength_config = strengthConfig_; + if (!frameContext.dpf.update && frame > 0) + return; + + auto config = params->block<BlockType::Dpf>(); + config.setEnabled(frameContext.dpf.denoise); + + if (frameContext.dpf.denoise) { + *config = config_; const auto &awb = context.configuration.awb; const auto &lsc = context.configuration.lsc; - auto &mode = params->others.dpf_config.gain.mode; + + auto &mode = config->gain.mode; /* * The DPF needs to take into account the total amount of @@ -241,15 +249,12 @@ void Dpf::prepare(IPAContext &context, const uint32_t frame, mode = RKISP1_CIF_ISP_DPF_GAIN_USAGE_LSC_GAINS; else mode = RKISP1_CIF_ISP_DPF_GAIN_USAGE_DISABLED; - - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_DPF | - RKISP1_CIF_ISP_MODULE_DPF_STRENGTH; } - if (frameContext.dpf.update) { - params->module_en_update |= RKISP1_CIF_ISP_MODULE_DPF; - if (frameContext.dpf.denoise) - params->module_ens |= RKISP1_CIF_ISP_MODULE_DPF; + if (frame == 0) { + auto strengthConfig = params->block<BlockType::DpfStrength>(); + strengthConfig.setEnabled(true); + *strengthConfig = strengthConfig_; } } diff --git a/src/ipa/rkisp1/algorithms/dpf.h b/src/ipa/rkisp1/algorithms/dpf.h index 58f29f74..2dd8cd36 100644 --- a/src/ipa/rkisp1/algorithms/dpf.h +++ b/src/ipa/rkisp1/algorithms/dpf.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * dpf.h - RkISP1 Denoise Pre-Filter control + * RkISP1 Denoise Pre-Filter control */ #pragma once @@ -27,7 +27,7 @@ public: const ControlList &controls) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) override; + RkISP1Params *params) override; private: struct rkisp1_cif_isp_dpf_config config_; diff --git a/src/ipa/rkisp1/algorithms/filter.cpp b/src/ipa/rkisp1/algorithms/filter.cpp index 4b89c05a..7598ef8a 100644 --- a/src/ipa/rkisp1/algorithms/filter.cpp +++ b/src/ipa/rkisp1/algorithms/filter.cpp @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * filter.cpp - RkISP1 Filter control + * RkISP1 Filter control */ #include "filter.h" @@ -104,7 +104,7 @@ void Filter::queueRequest(IPAContext &context, */ void Filter::prepare([[maybe_unused]] IPAContext &context, [[maybe_unused]] const uint32_t frame, - IPAFrameContext &frameContext, rkisp1_params_cfg *params) + IPAFrameContext &frameContext, RkISP1Params *params) { /* Check if the algorithm configuration has been updated. */ if (!frameContext.filter.update) @@ -160,23 +160,25 @@ void Filter::prepare([[maybe_unused]] IPAContext &context, uint8_t denoise = frameContext.filter.denoise; uint8_t sharpness = frameContext.filter.sharpness; - auto &flt_config = params->others.flt_config; - - flt_config.fac_sh0 = filt_fac_sh0[sharpness]; - flt_config.fac_sh1 = filt_fac_sh1[sharpness]; - flt_config.fac_mid = filt_fac_mid[sharpness]; - flt_config.fac_bl0 = filt_fac_bl0[sharpness]; - flt_config.fac_bl1 = filt_fac_bl1[sharpness]; - - flt_config.lum_weight = kFiltLumWeightDefault; - flt_config.mode = kFiltModeDefault; - flt_config.thresh_sh0 = filt_thresh_sh0[denoise]; - flt_config.thresh_sh1 = filt_thresh_sh1[denoise]; - flt_config.thresh_bl0 = filt_thresh_bl0[denoise]; - flt_config.thresh_bl1 = filt_thresh_bl1[denoise]; - flt_config.grn_stage1 = stage1_select[denoise]; - flt_config.chr_v_mode = filt_chr_v_mode[denoise]; - flt_config.chr_h_mode = filt_chr_h_mode[denoise]; + + auto config = params->block<BlockType::Flt>(); + config.setEnabled(true); + + config->fac_sh0 = filt_fac_sh0[sharpness]; + config->fac_sh1 = filt_fac_sh1[sharpness]; + config->fac_mid = filt_fac_mid[sharpness]; + config->fac_bl0 = filt_fac_bl0[sharpness]; + config->fac_bl1 = filt_fac_bl1[sharpness]; + + config->lum_weight = kFiltLumWeightDefault; + config->mode = kFiltModeDefault; + config->thresh_sh0 = filt_thresh_sh0[denoise]; + config->thresh_sh1 = filt_thresh_sh1[denoise]; + config->thresh_bl0 = filt_thresh_bl0[denoise]; + config->thresh_bl1 = filt_thresh_bl1[denoise]; + config->grn_stage1 = stage1_select[denoise]; + config->chr_v_mode = filt_chr_v_mode[denoise]; + config->chr_h_mode = filt_chr_h_mode[denoise]; /* * Combined high denoising and high sharpening requires some @@ -186,27 +188,23 @@ void Filter::prepare([[maybe_unused]] IPAContext &context, */ if (denoise == 9) { if (sharpness > 3) - flt_config.grn_stage1 = 2; + config->grn_stage1 = 2; } else if (denoise == 10) { if (sharpness > 5) - flt_config.grn_stage1 = 2; + config->grn_stage1 = 2; else if (sharpness > 3) - flt_config.grn_stage1 = 1; + config->grn_stage1 = 1; } if (denoise > 7) { if (sharpness > 7) { - flt_config.fac_bl0 /= 2; - flt_config.fac_bl1 /= 4; + config->fac_bl0 /= 2; + config->fac_bl1 /= 4; } else if (sharpness > 4) { - flt_config.fac_bl0 = flt_config.fac_bl0 * 3 / 4; - flt_config.fac_bl1 /= 2; + config->fac_bl0 = config->fac_bl0 * 3 / 4; + config->fac_bl1 /= 2; } } - - params->module_en_update |= RKISP1_CIF_ISP_MODULE_FLT; - params->module_ens |= RKISP1_CIF_ISP_MODULE_FLT; - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_FLT; } REGISTER_IPA_ALGORITHM(Filter, "Filter") diff --git a/src/ipa/rkisp1/algorithms/filter.h b/src/ipa/rkisp1/algorithms/filter.h index 3fd882ea..8f858e57 100644 --- a/src/ipa/rkisp1/algorithms/filter.h +++ b/src/ipa/rkisp1/algorithms/filter.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * filter.h - RkISP1 Filter control + * RkISP1 Filter control */ #pragma once @@ -26,7 +26,7 @@ public: const ControlList &controls) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) override; + RkISP1Params *params) override; }; } /* namespace ipa::rkisp1::algorithms */ diff --git a/src/ipa/rkisp1/algorithms/goc.cpp b/src/ipa/rkisp1/algorithms/goc.cpp new file mode 100644 index 00000000..a9493678 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/goc.cpp @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * RkISP1 Gamma out control + */ +#include "goc.h" + +#include <cmath> + +#include <libcamera/base/log.h> +#include <libcamera/base/utils.h> + +#include <libcamera/control_ids.h> + +#include "libcamera/internal/yaml_parser.h" + +#include "linux/rkisp1-config.h" + +/** + * \file goc.h + */ + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +/** + * \class GammaOutCorrection + * \brief RkISP1 Gamma out correction + * + * This algorithm implements the gamma out curve for the RkISP1. It defaults to + * a gamma value of 2.2. + * + * As gamma is internally represented as a piecewise linear function with only + * 17 knots, the difference between gamma=2.2 and sRGB gamma is minimal. + * Therefore sRGB gamma was not implemented as special case. + * + * Useful links: + * - https://www.cambridgeincolour.com/tutorials/gamma-correction.htm + * - https://en.wikipedia.org/wiki/SRGB + */ + +LOG_DEFINE_CATEGORY(RkISP1Gamma) + +const float kDefaultGamma = 2.2f; + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int GammaOutCorrection::init(IPAContext &context, const YamlObject &tuningData) +{ + if (context.hw->numGammaOutSamples != + RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES_V10) { + LOG(RkISP1Gamma, Error) + << "Gamma is not implemented for RkISP1 V12"; + return -EINVAL; + } + + defaultGamma_ = tuningData["gamma"].get<double>(kDefaultGamma); + context.ctrlMap[&controls::Gamma] = ControlInfo(0.1f, 10.0f, defaultGamma_); + + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::configure + */ +int GammaOutCorrection::configure(IPAContext &context, + [[maybe_unused]] const IPACameraSensorInfo &configInfo) +{ + context.activeState.goc.gamma = defaultGamma_; + return 0; +} + +/** + * \copydoc libcamera::ipa::Algorithm::queueRequest + */ +void GammaOutCorrection::queueRequest(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const ControlList &controls) +{ + if (frame == 0) + frameContext.goc.update = true; + + const auto &gamma = controls.get(controls::Gamma); + if (gamma) { + context.activeState.goc.gamma = *gamma; + frameContext.goc.update = true; + LOG(RkISP1Gamma, Debug) << "Set gamma to " << *gamma; + } + + frameContext.goc.gamma = context.activeState.goc.gamma; +} + +/** + * \copydoc libcamera::ipa::Algorithm::prepare + */ +void GammaOutCorrection::prepare(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + RkISP1Params *params) +{ + ASSERT(context.hw->numGammaOutSamples == + RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES_V10); + + if (!frameContext.goc.update) + return; + + /* + * The logarithmic segments as specified in the reference. + * Plus an additional 0 to make the loop easier + */ + static constexpr std::array<unsigned int, RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES_V10> segments = { + 64, 64, 64, 64, 128, 128, 128, 128, 256, + 256, 256, 512, 512, 512, 512, 512, 0 + }; + + auto config = params->block<BlockType::Goc>(); + config.setEnabled(true); + + __u16 *gamma_y = config->gamma_y; + + unsigned x = 0; + for (const auto [i, size] : utils::enumerate(segments)) { + gamma_y[i] = std::pow(x / 4096.0, 1.0 / frameContext.goc.gamma) * 1023.0; + x += size; + } + + config->mode = RKISP1_CIF_ISP_GOC_MODE_LOGARITHMIC; +} + +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void GammaOutCorrection::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + [[maybe_unused]] const rkisp1_stat_buffer *stats, + ControlList &metadata) +{ + metadata.set(controls::Gamma, frameContext.goc.gamma); +} + +REGISTER_IPA_ALGORITHM(GammaOutCorrection, "GammaOutCorrection") + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/goc.h b/src/ipa/rkisp1/algorithms/goc.h new file mode 100644 index 00000000..bb2ddfc9 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/goc.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * RkISP1 Gamma out control + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +class GammaOutCorrection : public Algorithm +{ +public: + GammaOutCorrection() = default; + ~GammaOutCorrection() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, + const IPACameraSensorInfo &configInfo) override; + void queueRequest(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + const ControlList &controls) override; + void prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + RkISP1Params *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats, + ControlList &metadata) override; + +private: + float defaultGamma_; +}; + +} /* namespace ipa::rkisp1::algorithms */ +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/gsl.cpp b/src/ipa/rkisp1/algorithms/gsl.cpp index b9f87912..9604c0ac 100644 --- a/src/ipa/rkisp1/algorithms/gsl.cpp +++ b/src/ipa/rkisp1/algorithms/gsl.cpp @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * gsl.cpp - RkISP1 Gamma Sensor Linearization control + * RkISP1 Gamma Sensor Linearization control */ #include "gsl.h" @@ -119,24 +119,20 @@ int GammaSensorLinearization::init([[maybe_unused]] IPAContext &context, void GammaSensorLinearization::prepare([[maybe_unused]] IPAContext &context, const uint32_t frame, [[maybe_unused]] IPAFrameContext &frameContext, - rkisp1_params_cfg *params) + RkISP1Params *params) { if (frame > 0) return; - params->others.sdg_config.xa_pnts.gamma_dx0 = gammaDx_[0]; - params->others.sdg_config.xa_pnts.gamma_dx1 = gammaDx_[1]; + auto config = params->block<BlockType::Sdg>(); + config.setEnabled(true); - std::copy(curveYr_.begin(), curveYr_.end(), - params->others.sdg_config.curve_r.gamma_y); - std::copy(curveYg_.begin(), curveYg_.end(), - params->others.sdg_config.curve_g.gamma_y); - std::copy(curveYb_.begin(), curveYb_.end(), - params->others.sdg_config.curve_b.gamma_y); + config->xa_pnts.gamma_dx0 = gammaDx_[0]; + config->xa_pnts.gamma_dx1 = gammaDx_[1]; - params->module_en_update |= RKISP1_CIF_ISP_MODULE_SDG; - params->module_ens |= RKISP1_CIF_ISP_MODULE_SDG; - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_SDG; + std::copy(curveYr_.begin(), curveYr_.end(), config->curve_r.gamma_y); + std::copy(curveYg_.begin(), curveYg_.end(), config->curve_g.gamma_y); + std::copy(curveYb_.begin(), curveYb_.end(), config->curve_b.gamma_y); } REGISTER_IPA_ALGORITHM(GammaSensorLinearization, "GammaSensorLinearization") diff --git a/src/ipa/rkisp1/algorithms/gsl.h b/src/ipa/rkisp1/algorithms/gsl.h index 0f1116a7..91cf6efa 100644 --- a/src/ipa/rkisp1/algorithms/gsl.h +++ b/src/ipa/rkisp1/algorithms/gsl.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * gsl.h - RkISP1 Gamma Sensor Linearization control + * RkISP1 Gamma Sensor Linearization control */ #pragma once @@ -22,7 +22,7 @@ public: int init(IPAContext &context, const YamlObject &tuningData) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) override; + RkISP1Params *params) override; private: uint32_t gammaDx_[2]; diff --git a/src/ipa/rkisp1/algorithms/lsc.cpp b/src/ipa/rkisp1/algorithms/lsc.cpp index a7ccedb1..e7301bfe 100644 --- a/src/ipa/rkisp1/algorithms/lsc.cpp +++ b/src/ipa/rkisp1/algorithms/lsc.cpp @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * lsc.cpp - RkISP1 Lens Shading Correction control + * RkISP1 Lens Shading Correction control */ #include "lsc.h" @@ -16,6 +16,7 @@ #include "libcamera/internal/yaml_parser.h" +#include "libipa/lsc_polynomial.h" #include "linux/rkisp1-config.h" /** @@ -24,6 +25,36 @@ namespace libcamera { +namespace ipa { + +constexpr int kColourTemperatureChangeThreshhold = 10; + +template<typename T> +void interpolateVector(const std::vector<T> &a, const std::vector<T> &b, + std::vector<T> &dest, double lambda) +{ + assert(a.size() == b.size()); + dest.resize(a.size()); + for (size_t i = 0; i < a.size(); i++) { + dest[i] = a[i] * (1.0 - lambda) + b[i] * lambda; + } +} + +template<> +void Interpolator<rkisp1::algorithms::LensShadingCorrection::Components>:: + interpolate(const rkisp1::algorithms::LensShadingCorrection::Components &a, + const rkisp1::algorithms::LensShadingCorrection::Components &b, + rkisp1::algorithms::LensShadingCorrection::Components &dest, + double lambda) +{ + interpolateVector(a.r, b.r, dest.r, lambda); + interpolateVector(a.gr, b.gr, dest.gr, lambda); + interpolateVector(a.gb, b.gb, dest.gb, lambda); + interpolateVector(a.b, b.b, dest.b, lambda); +} + +} /* namespace ipa */ + namespace ipa::rkisp1::algorithms { /** @@ -40,6 +71,200 @@ namespace ipa::rkisp1::algorithms { LOG_DEFINE_CATEGORY(RkISP1Lsc) +class LscPolynomialLoader +{ +public: + LscPolynomialLoader(const Size &sensorSize, + const Rectangle &cropRectangle, + const std::vector<double> &xSizes, + const std::vector<double> &ySizes) + : sensorSize_(sensorSize), + cropRectangle_(cropRectangle), + xSizes_(xSizes), + ySizes_(ySizes) + { + } + + int parseLscData(const YamlObject &yamlSets, + std::map<unsigned int, LensShadingCorrection::Components> &lscData) + { + const auto &sets = yamlSets.asList(); + for (const auto &yamlSet : sets) { + std::optional<LscPolynomial> pr, pgr, pgb, pb; + uint32_t ct = yamlSet["ct"].get<uint32_t>(0); + + if (lscData.count(ct)) { + LOG(RkISP1Lsc, Error) + << "Multiple sets found for " + << "color temperature " << ct; + return -EINVAL; + } + + LensShadingCorrection::Components &set = lscData[ct]; + pr = yamlSet["r"].get<LscPolynomial>(); + pgr = yamlSet["gr"].get<LscPolynomial>(); + pgb = yamlSet["gb"].get<LscPolynomial>(); + pb = yamlSet["b"].get<LscPolynomial>(); + + if (!(pr || pgr || pgb || pb)) { + LOG(RkISP1Lsc, Error) + << "Failed to parse polynomial for " + << "colour temperature " << ct; + return -EINVAL; + } + + set.ct = ct; + pr->setReferenceImageSize(sensorSize_); + pgr->setReferenceImageSize(sensorSize_); + pgb->setReferenceImageSize(sensorSize_); + pb->setReferenceImageSize(sensorSize_); + set.r = samplePolynomial(*pr); + set.gr = samplePolynomial(*pgr); + set.gb = samplePolynomial(*pgb); + set.b = samplePolynomial(*pb); + } + + if (lscData.empty()) { + LOG(RkISP1Lsc, Error) << "Failed to load any sets"; + return -EINVAL; + } + + return 0; + } + +private: + /* + * The lsc grid has custom spacing defined on half the range (see + * parseSizes() for details). For easier handling this function converts + * the spaces vector to positions and mirrors them. E.g.: + * + * input: | 0.2 | 0.3 | + * output: 0.0 0.2 0.5 0.8 1.0 + */ + std::vector<double> sizesListToPositions(const std::vector<double> &sizes) + { + const int half = sizes.size(); + std::vector<double> res(half * 2 + 1); + double x = 0.0; + + res[half] = 0.5; + for (int i = 1; i <= half; i++) { + x += sizes[half - i]; + res[half - i] = 0.5 - x; + res[half + i] = 0.5 + x; + } + + return res; + } + + std::vector<uint16_t> samplePolynomial(const LscPolynomial &poly) + { + constexpr int k = RKISP1_CIF_ISP_LSC_SAMPLES_MAX; + + double m = poly.getM(); + double x0 = cropRectangle_.x / m; + double y0 = cropRectangle_.y / m; + double w = cropRectangle_.width / m; + double h = cropRectangle_.height / m; + std::vector<uint16_t> res; + + assert(xSizes_.size() * 2 + 1 == k); + assert(ySizes_.size() * 2 + 1 == k); + + res.reserve(k * k); + + std::vector<double> xPos(sizesListToPositions(xSizes_)); + std::vector<double> yPos(sizesListToPositions(ySizes_)); + + for (int y = 0; y < k; y++) { + for (int x = 0; x < k; x++) { + double xp = x0 + xPos[x] * w; + double yp = y0 + yPos[y] * h; + /* + * The hardware uses 2.10 fixed point format and + * limits the legal values to [1..3.999]. Scale + * and clamp the sampled value accordingly. + */ + int v = static_cast<int>( + poly.sampleAtNormalizedPixelPos(xp, yp) * + 1024); + v = std::min(std::max(v, 1024), 4095); + res.push_back(v); + } + } + return res; + } + + Size sensorSize_; + Rectangle cropRectangle_; + const std::vector<double> &xSizes_; + const std::vector<double> &ySizes_; +}; + +class LscTableLoader +{ +public: + int parseLscData(const YamlObject &yamlSets, + std::map<unsigned int, LensShadingCorrection::Components> &lscData) + { + const auto &sets = yamlSets.asList(); + + for (const auto &yamlSet : sets) { + uint32_t ct = yamlSet["ct"].get<uint32_t>(0); + + if (lscData.count(ct)) { + LOG(RkISP1Lsc, Error) + << "Multiple sets found for color temperature " + << ct; + return -EINVAL; + } + + LensShadingCorrection::Components &set = lscData[ct]; + + set.ct = ct; + set.r = parseTable(yamlSet, "r"); + set.gr = parseTable(yamlSet, "gr"); + set.gb = parseTable(yamlSet, "gb"); + set.b = parseTable(yamlSet, "b"); + + if (set.r.empty() || set.gr.empty() || + set.gb.empty() || set.b.empty()) { + LOG(RkISP1Lsc, Error) + << "Set for color temperature " << ct + << " is missing tables"; + return -EINVAL; + } + } + + if (lscData.empty()) { + LOG(RkISP1Lsc, Error) << "Failed to load any sets"; + return -EINVAL; + } + + return 0; + } + +private: + std::vector<uint16_t> parseTable(const YamlObject &tuningData, + const char *prop) + { + static constexpr unsigned int kLscNumSamples = + RKISP1_CIF_ISP_LSC_SAMPLES_MAX * RKISP1_CIF_ISP_LSC_SAMPLES_MAX; + + std::vector<uint16_t> table = + tuningData[prop].getList<uint16_t>().value_or(std::vector<uint16_t>{}); + if (table.size() != kLscNumSamples) { + LOG(RkISP1Lsc, Error) + << "Invalid '" << prop << "' values: expected " + << kLscNumSamples + << " elements, got " << table.size(); + return {}; + } + + return table; + } +}; + static std::vector<double> parseSizes(const YamlObject &tuningData, const char *prop) { @@ -70,28 +295,10 @@ static std::vector<double> parseSizes(const YamlObject &tuningData, return sizes; } -static std::vector<uint16_t> parseTable(const YamlObject &tuningData, - const char *prop) -{ - static constexpr unsigned int kLscNumSamples = - RKISP1_CIF_ISP_LSC_SAMPLES_MAX * RKISP1_CIF_ISP_LSC_SAMPLES_MAX; - - std::vector<uint16_t> table = - tuningData[prop].getList<uint16_t>().value_or(std::vector<uint16_t>{}); - if (table.size() != kLscNumSamples) { - LOG(RkISP1Lsc, Error) - << "Invalid '" << prop << "' values: expected " - << kLscNumSamples - << " elements, got " << table.size(); - return {}; - } - - return table; -} - LensShadingCorrection::LensShadingCorrection() - : lastCt_({ 0, 0 }) + : lastAppliedCt_(0), lastAppliedQuantizedCt_(0) { + sets_.setQuantization(kColourTemperatureChangeThreshhold); } /** @@ -114,38 +321,30 @@ int LensShadingCorrection::init([[maybe_unused]] IPAContext &context, return -EINVAL; } - const auto &sets = yamlSets.asList(); - for (const auto &yamlSet : sets) { - uint32_t ct = yamlSet["ct"].get<uint32_t>(0); - - if (sets_.count(ct)) { - LOG(RkISP1Lsc, Error) - << "Multiple sets found for color temperature " - << ct; - return -EINVAL; - } - - Components &set = sets_[ct]; - - set.ct = ct; - set.r = parseTable(yamlSet, "r"); - set.gr = parseTable(yamlSet, "gr"); - set.gb = parseTable(yamlSet, "gb"); - set.b = parseTable(yamlSet, "b"); - - if (set.r.empty() || set.gr.empty() || - set.gb.empty() || set.b.empty()) { - LOG(RkISP1Lsc, Error) - << "Set for color temperature " << ct - << " is missing tables"; - return -EINVAL; - } + std::map<unsigned int, Components> lscData; + int res = 0; + std::string type = tuningData["type"].get<std::string>("table"); + if (type == "table") { + LOG(RkISP1Lsc, Debug) << "Loading tabular LSC data."; + auto loader = LscTableLoader(); + res = loader.parseLscData(yamlSets, lscData); + } else if (type == "polynomial") { + LOG(RkISP1Lsc, Debug) << "Loading polynomial LSC data."; + auto loader = LscPolynomialLoader(context.sensorInfo.activeAreaSize, + context.sensorInfo.analogCrop, + xSize_, + ySize_); + res = loader.parseLscData(yamlSets, lscData); + } else { + LOG(RkISP1Lsc, Error) << "Unsupported LSC data type '" + << type << "'"; + res = -EINVAL; } - if (sets_.empty()) { - LOG(RkISP1Lsc, Error) << "Failed to load any sets"; - return -EINVAL; - } + if (res) + return res; + + sets_.setData(std::move(lscData)); return 0; } @@ -185,18 +384,12 @@ int LensShadingCorrection::configure(IPAContext &context, return 0; } -void LensShadingCorrection::setParameters(rkisp1_params_cfg *params) +void LensShadingCorrection::setParameters(rkisp1_cif_isp_lsc_config &config) { - struct rkisp1_cif_isp_lsc_config &config = params->others.lsc_config; - memcpy(config.x_grad_tbl, xGrad_, sizeof(config.x_grad_tbl)); memcpy(config.y_grad_tbl, yGrad_, sizeof(config.y_grad_tbl)); memcpy(config.x_size_tbl, xSizes_, sizeof(config.x_size_tbl)); memcpy(config.y_size_tbl, ySizes_, sizeof(config.y_size_tbl)); - - params->module_en_update |= RKISP1_CIF_ISP_MODULE_LSC; - params->module_ens |= RKISP1_CIF_ISP_MODULE_LSC; - params->module_cfg_update |= RKISP1_CIF_ISP_MODULE_LSC; } void LensShadingCorrection::copyTable(rkisp1_cif_isp_lsc_config &config, @@ -208,131 +401,34 @@ void LensShadingCorrection::copyTable(rkisp1_cif_isp_lsc_config &config, std::copy(set.b.begin(), set.b.end(), &config.b_data_tbl[0][0]); } -/* - * Interpolate LSC parameters based on color temperature value. - */ -void LensShadingCorrection::interpolateTable(rkisp1_cif_isp_lsc_config &config, - const Components &set0, - const Components &set1, - const uint32_t ct) -{ - double coeff0 = (set1.ct - ct) / static_cast<double>(set1.ct - set0.ct); - double coeff1 = (ct - set0.ct) / static_cast<double>(set1.ct - set0.ct); - - for (unsigned int i = 0; i < RKISP1_CIF_ISP_LSC_SAMPLES_MAX; ++i) { - for (unsigned int j = 0; j < RKISP1_CIF_ISP_LSC_SAMPLES_MAX; ++j) { - unsigned int sample = i * RKISP1_CIF_ISP_LSC_SAMPLES_MAX + j; - - config.r_data_tbl[i][j] = - set0.r[sample] * coeff0 + - set1.r[sample] * coeff1; - - config.gr_data_tbl[i][j] = - set0.gr[sample] * coeff0 + - set1.gr[sample] * coeff1; - - config.gb_data_tbl[i][j] = - set0.gb[sample] * coeff0 + - set1.gb[sample] * coeff1; - - config.b_data_tbl[i][j] = - set0.b[sample] * coeff0 + - set1.b[sample] * coeff1; - } - } -} - /** * \copydoc libcamera::ipa::Algorithm::prepare */ -void LensShadingCorrection::prepare(IPAContext &context, - const uint32_t frame, - [[maybe_unused]] IPAFrameContext &frameContext, - rkisp1_params_cfg *params) +void LensShadingCorrection::prepare([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + RkISP1Params *params) { - struct rkisp1_cif_isp_lsc_config &config = params->others.lsc_config; - - /* - * If there is only one set, the configuration has already been done - * for first frame. - */ - if (sets_.size() == 1 && frame > 0) + uint32_t ct = frameContext.awb.temperatureK; + if (std::abs(static_cast<int>(ct) - static_cast<int>(lastAppliedCt_)) < + kColourTemperatureChangeThreshhold) return; - - /* - * If there is only one set, pick it. We can ignore lastCt_, as it will - * never be relevant. - */ - if (sets_.size() == 1) { - setParameters(params); - copyTable(config, sets_.cbegin()->second); - return; - } - - uint32_t ct = context.activeState.awb.temperatureK; - ct = std::clamp(ct, sets_.cbegin()->first, sets_.crbegin()->first); - - /* - * If the original is the same, then it means the same adjustment would - * be made. If the adjusted is the same, then it means that it's the - * same as what was actually applied. Thus in these cases we can skip - * reprogramming the LSC. - * - * original == adjusted can only happen if an interpolation - * happened, or if original has an exact entry in sets_. This means - * that if original != adjusted, then original was adjusted to - * the nearest available entry in sets_, resulting in adjusted. - * Clearly, any ct value that is in between original and adjusted - * will be adjusted to the same adjusted value, so we can skip - * reprogramming the LSC table. - * - * We also skip updating the original value, as the last one had a - * larger bound and thus a larger range of ct values that will be - * adjusted to the same adjusted. - */ - if ((lastCt_.original <= ct && ct <= lastCt_.adjusted) || - (lastCt_.adjusted <= ct && ct <= lastCt_.original)) + unsigned int quantizedCt; + const Components &set = sets_.getInterpolated(ct, &quantizedCt); + if (lastAppliedQuantizedCt_ == quantizedCt) return; - setParameters(params); + auto config = params->block<BlockType::Lsc>(); + config.setEnabled(true); + setParameters(*config); + copyTable(*config, set); - /* - * The color temperature matches exactly one of the available LSC tables. - */ - if (sets_.count(ct)) { - copyTable(config, sets_[ct]); - lastCt_ = { ct, ct }; - return; - } - - /* No shortcuts left; we need to round or interpolate */ - auto iter = sets_.upper_bound(ct); - const Components &set1 = iter->second; - const Components &set0 = (--iter)->second; - uint32_t ct0 = set0.ct; - uint32_t ct1 = set1.ct; - uint32_t diff0 = ct - ct0; - uint32_t diff1 = ct1 - ct; - static constexpr double kThreshold = 0.1; - float threshold = kThreshold * (ct1 - ct0); - - if (diff0 < threshold || diff1 < threshold) { - const Components &set = diff0 < diff1 ? set0 : set1; - LOG(RkISP1Lsc, Debug) << "using LSC table for " << set.ct; - copyTable(config, set); - lastCt_ = { ct, set.ct }; - return; - } + lastAppliedCt_ = ct; + lastAppliedQuantizedCt_ = quantizedCt; - /* - * ct is not within 10% of the difference between the neighbouring - * color temperatures, so we need to interpolate. - */ LOG(RkISP1Lsc, Debug) - << "ct is " << ct << ", interpolating between " - << ct0 << " and " << ct1; - interpolateTable(config, set0, set1, ct); - lastCt_ = { ct, ct }; + << "ct is " << ct << ", quantized to " + << quantizedCt; } REGISTER_IPA_ALGORITHM(LensShadingCorrection, "LensShadingCorrection") diff --git a/src/ipa/rkisp1/algorithms/lsc.h b/src/ipa/rkisp1/algorithms/lsc.h index e2a93a56..5a0824e3 100644 --- a/src/ipa/rkisp1/algorithms/lsc.h +++ b/src/ipa/rkisp1/algorithms/lsc.h @@ -2,13 +2,15 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * lsc.h - RkISP1 Lens Shading Correction control + * RkISP1 Lens Shading Correction control */ #pragma once #include <map> +#include "libipa/interpolator.h" + #include "algorithm.h" namespace libcamera { @@ -25,9 +27,8 @@ public: int configure(IPAContext &context, const IPACameraSensorInfo &configInfo) override; void prepare(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, - rkisp1_params_cfg *params) override; + RkISP1Params *params) override; -private: struct Components { uint32_t ct; std::vector<uint16_t> r; @@ -36,23 +37,23 @@ private: std::vector<uint16_t> b; }; - void setParameters(rkisp1_params_cfg *params); +private: + void setParameters(rkisp1_cif_isp_lsc_config &config); void copyTable(rkisp1_cif_isp_lsc_config &config, const Components &set0); void interpolateTable(rkisp1_cif_isp_lsc_config &config, const Components &set0, const Components &set1, const uint32_t ct); - std::map<uint32_t, Components> sets_; + ipa::Interpolator<Components> sets_; std::vector<double> xSize_; std::vector<double> ySize_; uint16_t xGrad_[RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE]; uint16_t yGrad_[RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE]; uint16_t xSizes_[RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE]; uint16_t ySizes_[RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE]; - struct { - uint32_t original; - uint32_t adjusted; - } lastCt_; + + unsigned int lastAppliedCt_; + unsigned int lastAppliedQuantizedCt_; }; } /* namespace ipa::rkisp1::algorithms */ diff --git a/src/ipa/rkisp1/algorithms/lux.cpp b/src/ipa/rkisp1/algorithms/lux.cpp new file mode 100644 index 00000000..a467767e --- /dev/null +++ b/src/ipa/rkisp1/algorithms/lux.cpp @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * lux.cpp - RkISP1 Lux control + */ + +#include "lux.h" + +#include <libcamera/base/log.h> + +#include <libcamera/control_ids.h> + +#include "libipa/histogram.h" +#include "libipa/lux.h" + +/** + * \file lux.h + */ + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +/** + * \class Lux + * \brief RkISP1 Lux control + * + * The Lux algorithm is responsible for estimating the lux level of the image. + * It doesn't take or generate any controls, but it provides a lux level for + * other algorithms (such as AGC) to use. + */ + +/** + * \brief Construct an rkisp1 Lux algo module + */ +Lux::Lux() +{ +} + +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Lux::init([[maybe_unused]] IPAContext &context, const YamlObject &tuningData) +{ + return lux_.parseTuningData(tuningData); +} + +/** + * \copydoc libcamera::ipa::Algorithm::process + */ +void Lux::process(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats, + ControlList &metadata) +{ + utils::Duration exposureTime = context.configuration.sensor.lineDuration + * frameContext.sensor.exposure; + double gain = frameContext.sensor.gain; + + /* \todo Deduplicate the histogram calculation from AGC */ + const rkisp1_cif_isp_stat *params = &stats->params; + Histogram yHist({ params->hist.hist_bins, context.hw->numHistogramBins }, + [](uint32_t x) { return x >> 4; }); + + double lux = lux_.estimateLux(exposureTime, gain, 1.0, yHist); + frameContext.lux.lux = lux; + metadata.set(controls::Lux, lux); +} + +REGISTER_IPA_ALGORITHM(Lux, "Lux") + +} /* namespace ipa::rkisp1::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/lux.h b/src/ipa/rkisp1/algorithms/lux.h new file mode 100644 index 00000000..8a90de55 --- /dev/null +++ b/src/ipa/rkisp1/algorithms/lux.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * lux.h - RkISP1 Lux control + */ + +#pragma once + +#include <sys/types.h> + +#include "libipa/lux.h" + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::rkisp1::algorithms { + +class Lux : public Algorithm +{ +public: + Lux(); + + int init(IPAContext &context, const YamlObject &tuningData) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const rkisp1_stat_buffer *stats, + ControlList &metadata) override; + +private: + ipa::Lux lux_; +}; + +} /* namespace ipa::rkisp1::algorithms */ +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/algorithms/meson.build b/src/ipa/rkisp1/algorithms/meson.build index 93a48329..c66b0b70 100644 --- a/src/ipa/rkisp1/algorithms/meson.build +++ b/src/ipa/rkisp1/algorithms/meson.build @@ -4,10 +4,13 @@ rkisp1_ipa_algorithms = files([ 'agc.cpp', 'awb.cpp', 'blc.cpp', + 'ccm.cpp', 'cproc.cpp', 'dpcc.cpp', 'dpf.cpp', 'filter.cpp', + 'goc.cpp', 'gsl.cpp', 'lsc.cpp', + 'lux.cpp', ]) diff --git a/src/ipa/rkisp1/data/imx219.yaml b/src/ipa/rkisp1/data/imx219.yaml index cbcc43b8..0d99cb52 100644 --- a/src/ipa/rkisp1/data/imx219.yaml +++ b/src/ipa/rkisp1/data/imx219.yaml @@ -6,10 +6,6 @@ algorithms: - Agc: - Awb: - BlackLevelCorrection: - R: 256 - Gr: 256 - Gb: 256 - B: 256 - LensShadingCorrection: x-size: [ 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625 ] y-size: [ 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625 ] diff --git a/src/ipa/rkisp1/data/imx258.yaml b/src/ipa/rkisp1/data/imx258.yaml index 43dddf20..202af36a 100644 --- a/src/ipa/rkisp1/data/imx258.yaml +++ b/src/ipa/rkisp1/data/imx258.yaml @@ -5,6 +5,7 @@ version: 1 algorithms: - Agc: - Awb: + - BlackLevelCorrection: - LensShadingCorrection: x-size: [ 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625 ] y-size: [ 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625 ] diff --git a/src/ipa/rkisp1/data/meson.build b/src/ipa/rkisp1/data/meson.build index f5e9fa75..1e3522b2 100644 --- a/src/ipa/rkisp1/data/meson.build +++ b/src/ipa/rkisp1/data/meson.build @@ -2,10 +2,15 @@ conf_files = files([ 'imx219.yaml', + 'imx258.yaml', + 'ov2685.yaml', 'ov4689.yaml', 'ov5640.yaml', + 'ov5695.yaml', + 'ov8858.yaml', 'uncalibrated.yaml', ]) install_data(conf_files, - install_dir : ipa_data_dir / 'rkisp1') + install_dir : ipa_data_dir / 'rkisp1', + install_tag : 'runtime') diff --git a/src/ipa/rkisp1/data/ov4689.yaml b/src/ipa/rkisp1/data/ov4689.yaml index 2068684c..60901296 100644 --- a/src/ipa/rkisp1/data/ov4689.yaml +++ b/src/ipa/rkisp1/data/ov4689.yaml @@ -6,8 +6,4 @@ algorithms: - Agc: - Awb: - BlackLevelCorrection: - R: 66 - Gr: 66 - Gb: 66 - B: 66 ... diff --git a/src/ipa/rkisp1/data/ov5640.yaml b/src/ipa/rkisp1/data/ov5640.yaml index 897b83cb..4b21d412 100644 --- a/src/ipa/rkisp1/data/ov5640.yaml +++ b/src/ipa/rkisp1/data/ov5640.yaml @@ -6,10 +6,6 @@ algorithms: - Agc: - Awb: - BlackLevelCorrection: - R: 256 - Gr: 256 - Gb: 256 - B: 256 - ColorProcessing: - GammaSensorLinearization: x-intervals: [ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 ] diff --git a/src/ipa/rkisp1/data/uncalibrated.yaml b/src/ipa/rkisp1/data/uncalibrated.yaml index a7bbd8d8..60901296 100644 --- a/src/ipa/rkisp1/data/uncalibrated.yaml +++ b/src/ipa/rkisp1/data/uncalibrated.yaml @@ -5,4 +5,5 @@ version: 1 algorithms: - Agc: - Awb: + - BlackLevelCorrection: ... diff --git a/src/ipa/rkisp1/ipa_context.cpp b/src/ipa/rkisp1/ipa_context.cpp index 9bbf3684..65096105 100644 --- a/src/ipa/rkisp1/ipa_context.cpp +++ b/src/ipa/rkisp1/ipa_context.cpp @@ -2,7 +2,7 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * ipa_context.cpp - RkISP1 IPA Context + * RkISP1 IPA Context */ #include "ipa_context.h" @@ -15,6 +15,25 @@ namespace libcamera::ipa::rkisp1 { /** + * \struct IPAHwSettings + * \brief RkISP1 version-specific hardware parameters + */ + +/** + * \var IPAHwSettings::numAeCells + * \brief Number of cells in the AE exposure means grid + * + * \var IPAHwSettings::numHistogramBins + * \brief Number of bins in the histogram + * + * \var IPAHwSettings::numHistogramWeights + * \brief Number of weights in the histogram grid + * + * \var IPAHwSettings::numGammaOutSamples + * \brief Number of samples in the gamma out table + */ + +/** * \struct IPASessionConfiguration * \brief Session configuration for the IPA module * @@ -33,14 +52,6 @@ namespace libcamera::ipa::rkisp1 { */ /** - * \var IPASessionConfiguration::hw - * \brief RkISP1-specific hardware information - * - * \var IPASessionConfiguration::hw.revision - * \brief Hardware revision of the ISP - */ - -/** * \var IPASessionConfiguration::awb * \brief AWB parameters configuration of the IPA * @@ -67,11 +78,11 @@ namespace libcamera::ipa::rkisp1 { * \var IPASessionConfiguration::sensor * \brief Sensor-specific configuration of the IPA * - * \var IPASessionConfiguration::sensor.minShutterSpeed - * \brief Minimum shutter speed supported with the sensor + * \var IPASessionConfiguration::sensor.minExposureTime + * \brief Minimum exposure time supported with the sensor * - * \var IPASessionConfiguration::sensor.maxShutterSpeed - * \brief Maximum shutter speed supported with the sensor + * \var IPASessionConfiguration::sensor.maxExposureTime + * \brief Maximum exposure time supported with the sensor * * \var IPASessionConfiguration::sensor.minAnalogueGain * \brief Minimum analogue gain supported with the sensor @@ -95,6 +106,11 @@ namespace libcamera::ipa::rkisp1 { */ /** + * \var IPASessionConfiguration::paramFormat + * \brief The fourcc of the parameters buffers format + */ + +/** * \struct IPAActiveState * \brief Active state for algorithms * @@ -126,54 +142,86 @@ namespace libcamera::ipa::rkisp1 { * \var IPAActiveState::agc * \brief State for the Automatic Gain Control algorithm * - * The exposure and gain are the latest values computed by the AGC algorithm. + * The \a automatic variables track the latest values computed by algorithm + * based on the latest processed statistics. All other variables track the + * consolidated controls requested in queued requests. * - * \var IPAActiveState::agc.exposure - * \brief Exposure time expressed as a number of lines + * \struct IPAActiveState::agc.manual + * \brief Manual exposure time and analog gain (set through requests) * - * \var IPAActiveState::agc.gain - * \brief Analogue gain multiplier - */ - -/** - * \var IPAActiveState::awb - * \brief State for the Automatic White Balance algorithm + * \var IPAActiveState::agc.manual.exposure + * \brief Manual exposure time expressed as a number of lines as set by the + * ExposureTime control * - * \struct IPAActiveState::awb.gains - * \brief White balance gains + * \var IPAActiveState::agc.manual.gain + * \brief Manual analogue gain as set by the AnalogueGain control * - * \struct IPAActiveState::awb.gains.manual - * \brief Manual white balance gains (set through requests) + * \struct IPAActiveState::agc.automatic + * \brief Automatic exposure time and analog gain (computed by the algorithm) * - * \var IPAActiveState::awb.gains.manual.red - * \brief Manual white balance gain for R channel + * \var IPAActiveState::agc.automatic.exposure + * \brief Automatic exposure time expressed as a number of lines * - * \var IPAActiveState::awb.gains.manual.green - * \brief Manual white balance gain for G channel + * \var IPAActiveState::agc.automatic.gain + * \brief Automatic analogue gain multiplier * - * \var IPAActiveState::awb.gains.manual.blue - * \brief Manual white balance gain for B channel + * \var IPAActiveState::agc.autoExposureEnabled + * \brief Manual/automatic AGC state (exposure) as set by the ExposureTimeMode control * - * \struct IPAActiveState::awb.gains.automatic - * \brief Automatic white balance gains (computed by the algorithm) + * \var IPAActiveState::agc.autoGainEnabled + * \brief Manual/automatic AGC state (gain) as set by the AnalogueGainMode control * - * \var IPAActiveState::awb.gains.automatic.red - * \brief Automatic white balance gain for R channel + * \var IPAActiveState::agc.constraintMode + * \brief Constraint mode as set by the AeConstraintMode control * - * \var IPAActiveState::awb.gains.automatic.green - * \brief Automatic white balance gain for G channel + * \var IPAActiveState::agc.exposureMode + * \brief Exposure mode as set by the AeExposureMode control * - * \var IPAActiveState::awb.gains.automatic.blue - * \brief Automatic white balance gain for B channel + * \var IPAActiveState::agc.meteringMode + * \brief Metering mode as set by the AeMeteringMode control * - * \var IPAActiveState::awb.temperatureK - * \brief Estimated color temperature + * \var IPAActiveState::agc.minFrameDuration + * \brief Minimum frame duration as set by the FrameDurationLimits control + * + * \var IPAActiveState::agc.maxFrameDuration + * \brief Maximum frame duration as set by the FrameDurationLimits control + */ + +/** + * \var IPAActiveState::awb + * \brief State for the Automatic White Balance algorithm + * + * \struct IPAActiveState::awb::AwbState + * \brief Struct for the AWB regulation state + * + * \var IPAActiveState::awb::AwbState.gains + * \brief White balance gains + * + * \var IPAActiveState::awb::AwbState.temperatureK + * \brief Color temperature + * + * \var IPAActiveState::awb.manual + * \brief Manual regulation state (set through requests) + * + * \var IPAActiveState::awb.automatic + * \brief Automatic regulation state (computed by the algorithm) * * \var IPAActiveState::awb.autoEnabled * \brief Whether the Auto White Balance algorithm is enabled */ /** + * \var IPAActiveState::ccm + * \brief State for the Colour Correction Matrix algorithm + * + * \var IPAActiveState::ccm.manual + * \brief Manual CCM (set through requests) + * + * \var IPAActiveState::awb.automatic + * \brief Automatic CCM (computed by the algorithm) + */ + +/** * \var IPAActiveState::cproc * \brief State for the Color Processing algorithm * @@ -207,6 +255,14 @@ namespace libcamera::ipa::rkisp1 { */ /** + * \var IPAActiveState::goc + * \brief State for the goc algorithm + * + * \var IPAActiveState::goc.gamma + * \brief Gamma value applied as 1.0/gamma + */ + +/** * \struct IPAFrameContext * \brief Per-frame context for algorithms * @@ -243,15 +299,57 @@ namespace libcamera::ipa::rkisp1 { * \brief Automatic Gain Control parameters for this frame * * The exposure and gain are provided by the AGC algorithm, and are to be - * applied to the sensor in order to take effect for this frame. + * applied to the sensor in order to take effect for this frame. Additionally + * the vertical blanking period is determined to maintain a consistent frame + * rate matched to the FrameDurationLimits as set by the user. * * \var IPAFrameContext::agc.exposure - * \brief Exposure time expressed as a number of lines + * \brief Exposure time expressed as a number of lines computed by the algorithm * * \var IPAFrameContext::agc.gain - * \brief Analogue gain multiplier + * \brief Analogue gain multiplier computed by the algorithm * * The gain should be adapted to the sensor specific gain code before applying. + * + * \var IPAFrameContext::agc.vblank + * \brief Vertical blanking parameter computed by the algorithm + * + * \var IPAFrameContext::agc.autoExposureEnabled + * \brief Manual/automatic AGC state (exposure) as set by the ExposureTimeMode control + * + * \var IPAFrameContext::agc.autoGainEnabled + * \brief Manual/automatic AGC state (gain) as set by the AnalogueGainMode control + * + * \var IPAFrameContext::agc.constraintMode + * \brief Constraint mode as set by the AeConstraintMode control + * + * \var IPAFrameContext::agc.exposureMode + * \brief Exposure mode as set by the AeExposureMode control + * + * \var IPAFrameContext::agc.meteringMode + * \brief Metering mode as set by the AeMeteringMode control + * + * \var IPAFrameContext::agc.minFrameDuration + * \brief Minimum frame duration as set by the FrameDurationLimits control + * + * \var IPAFrameContext::agc.maxFrameDuration + * \brief Maximum frame duration as set by the FrameDurationLimits control + * + * \var IPAFrameContext::agc.frameDuration + * \brief The actual FrameDuration used by the algorithm for the frame + * + * \var IPAFrameContext::agc.updateMetering + * \brief Indicate if new ISP AGC metering parameters need to be applied + * + * \var IPAFrameContext::agc.autoExposureModeChange + * \brief Indicate if autoExposureEnabled has changed from true in the previous + * frame to false in the current frame, and no manual exposure value has been + * supplied in the current frame. + * + * \var IPAFrameContext::agc.autoGainModeChange + * \brief Indicate if autoGainEnabled has changed from true in the previous + * frame to false in the current frame, and no manual gain value has been + * supplied in the current frame. */ /** @@ -261,23 +359,25 @@ namespace libcamera::ipa::rkisp1 { * \struct IPAFrameContext::awb.gains * \brief White balance gains * - * \var IPAFrameContext::awb.gains.red - * \brief White balance gain for R channel - * - * \var IPAFrameContext::awb.gains.green - * \brief White balance gain for G channel - * - * \var IPAFrameContext::awb.gains.blue - * \brief White balance gain for B channel - * * \var IPAFrameContext::awb.temperatureK - * \brief Estimated color temperature + * \brief Color temperature used for processing this frame + * + * This does not match the color temperature estimated for this frame as the + * measurements were taken on a previous frame. * * \var IPAFrameContext::awb.autoEnabled * \brief Whether the Auto White Balance algorithm is enabled */ /** + * \var IPAFrameContext::ccm + * \brief Colour Correction Matrix parameters for this frame + * + * \struct IPAFrameContext::ccm.ccm + * \brief Colour Correction Matrix + */ + +/** * \var IPAFrameContext::cproc * \brief Color Processing parameters for this frame * @@ -323,6 +423,18 @@ namespace libcamera::ipa::rkisp1 { */ /** + * \var IPAFrameContext::goc + * \brief Gamma out correction parameters for this frame + * + * \var IPAFrameContext::goc.gamma + * \brief Gamma value applied as 1.0/gamma + * + * \var IPAFrameContext::goc.update + * \brief Indicates if the goc parameters have been updated compared to the + * previous frame + */ + +/** * \var IPAFrameContext::sensor * \brief Sensor configuration that used been used for this frame * @@ -337,6 +449,12 @@ namespace libcamera::ipa::rkisp1 { * \struct IPAContext * \brief Global IPA context data shared between all algorithms * + * \var IPAContext::hw + * \brief RkISP1 version-specific hardware parameters + * + * \var IPAContext::sensorInfo + * \brief The IPA session sensorInfo, immutable during the session + * * \var IPAContext::configuration * \brief The IPA session configuration, immutable during the session * diff --git a/src/ipa/rkisp1/ipa_context.h b/src/ipa/rkisp1/ipa_context.h index b9b20653..f0d50421 100644 --- a/src/ipa/rkisp1/ipa_context.h +++ b/src/ipa/rkisp1/ipa_context.h @@ -2,24 +2,43 @@ /* * Copyright (C) 2021-2022, Ideas On Board * - * ipa_context.h - RkISP1 IPA Context + * RkISP1 IPA Context * */ #pragma once +#include <memory> + #include <linux/rkisp1-config.h> #include <libcamera/base/utils.h> +#include <libcamera/control_ids.h> +#include <libcamera/controls.h> #include <libcamera/geometry.h> +#include <libcamera/ipa/core_ipa_interface.h> + +#include "libcamera/internal/debug_controls.h" +#include "libcamera/internal/matrix.h" +#include "libcamera/internal/vector.h" + +#include <libipa/camera_sensor_helper.h> #include <libipa/fc_queue.h> namespace libcamera { namespace ipa::rkisp1 { +struct IPAHwSettings { + unsigned int numAeCells; + unsigned int numHistogramBins; + unsigned int numHistogramWeights; + unsigned int numGammaOutSamples; + bool compand; +}; + struct IPASessionConfiguration { struct { struct rkisp1_cif_isp_window measureWindow; @@ -35,8 +54,8 @@ struct IPASessionConfiguration { } lsc; struct { - utils::Duration minShutterSpeed; - utils::Duration maxShutterSpeed; + utils::Duration minExposureTime; + utils::Duration maxExposureTime; double minAnalogueGain; double maxAnalogueGain; @@ -45,11 +64,8 @@ struct IPASessionConfiguration { Size size; } sensor; - struct { - rkisp1_cif_isp_version revision; - } hw; - bool raw; + uint32_t paramFormat; }; struct IPAActiveState { @@ -63,28 +79,33 @@ struct IPAActiveState { double gain; } automatic; - bool autoEnabled; + bool autoExposureEnabled; + bool autoGainEnabled; + controls::AeConstraintModeEnum constraintMode; + controls::AeExposureModeEnum exposureMode; + controls::AeMeteringModeEnum meteringMode; + utils::Duration minFrameDuration; + utils::Duration maxFrameDuration; } agc; struct { - struct { - struct { - double red; - double green; - double blue; - } manual; - struct { - double red; - double green; - double blue; - } automatic; - } gains; + struct AwbState { + RGB<double> gains; + unsigned int temperatureK; + }; + + AwbState manual; + AwbState automatic; - unsigned int temperatureK; bool autoEnabled; } awb; struct { + Matrix<float, 3, 3> manual; + Matrix<float, 3, 3> automatic; + } ccm; + + struct { int8_t brightness; uint8_t contrast; uint8_t saturation; @@ -98,24 +119,34 @@ struct IPAActiveState { uint8_t denoise; uint8_t sharpness; } filter; + + struct { + double gamma; + } goc; }; struct IPAFrameContext : public FrameContext { struct { uint32_t exposure; double gain; - bool autoEnabled; + uint32_t vblank; + bool autoExposureEnabled; + bool autoGainEnabled; + controls::AeConstraintModeEnum constraintMode; + controls::AeExposureModeEnum exposureMode; + controls::AeMeteringModeEnum meteringMode; + utils::Duration minFrameDuration; + utils::Duration maxFrameDuration; + utils::Duration frameDuration; + bool updateMetering; + bool autoExposureModeChange; + bool autoGainModeChange; } agc; struct { - struct { - double red; - double green; - double blue; - } gains; - - unsigned int temperatureK; + RGB<double> gains; bool autoEnabled; + unsigned int temperatureK; } awb; struct { @@ -137,16 +168,43 @@ struct IPAFrameContext : public FrameContext { } filter; struct { + double gamma; + bool update; + } goc; + + struct { uint32_t exposure; double gain; } sensor; + + struct { + Matrix<float, 3, 3> ccm; + } ccm; + + struct { + double lux; + } lux; }; struct IPAContext { + IPAContext(unsigned int frameContextSize) + : hw(nullptr), frameContexts(frameContextSize) + { + } + + const IPAHwSettings *hw; + IPACameraSensorInfo sensorInfo; IPASessionConfiguration configuration; IPAActiveState activeState; FCQueue<IPAFrameContext> frameContexts; + + ControlInfoMap::Map ctrlMap; + + DebugMetadata debugMetadata; + + /* Interface to the Camera Helper */ + std::unique_ptr<CameraSensorHelper> camHelper; }; } /* namespace ipa::rkisp1 */ diff --git a/src/ipa/rkisp1/meson.build b/src/ipa/rkisp1/meson.build index e813da53..26a9fa40 100644 --- a/src/ipa/rkisp1/meson.build +++ b/src/ipa/rkisp1/meson.build @@ -7,17 +7,16 @@ ipa_name = 'ipa_rkisp1' rkisp1_ipa_sources = files([ 'ipa_context.cpp', + 'params.cpp', 'rkisp1.cpp', ]) rkisp1_ipa_sources += rkisp1_ipa_algorithms -mod = shared_module(ipa_name, - [rkisp1_ipa_sources, libcamera_generated_ipa_headers], +mod = shared_module(ipa_name, rkisp1_ipa_sources, name_prefix : '', - include_directories : [ipa_includes, libipa_includes], - dependencies : libcamera_private, - link_with : libipa, + include_directories : [ipa_includes], + dependencies : [libcamera_private, libipa_dep], install : true, install_dir : ipa_install_dir) diff --git a/src/ipa/rkisp1/module.h b/src/ipa/rkisp1/module.h index 89f83208..69e9bc82 100644 --- a/src/ipa/rkisp1/module.h +++ b/src/ipa/rkisp1/module.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2022, Ideas On Board * - * module.h - RkISP1 IPA Module + * RkISP1 IPA Module */ #pragma once @@ -14,13 +14,14 @@ #include <libipa/module.h> #include "ipa_context.h" +#include "params.h" namespace libcamera { namespace ipa::rkisp1 { using Module = ipa::Module<IPAContext, IPAFrameContext, IPACameraSensorInfo, - rkisp1_params_cfg, rkisp1_stat_buffer>; + RkISP1Params, rkisp1_stat_buffer>; } /* namespace ipa::rkisp1 */ diff --git a/src/ipa/rkisp1/params.cpp b/src/ipa/rkisp1/params.cpp new file mode 100644 index 00000000..4c0b051c --- /dev/null +++ b/src/ipa/rkisp1/params.cpp @@ -0,0 +1,222 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * RkISP1 ISP Parameters + */ + +#include "params.h" + +#include <map> +#include <stddef.h> +#include <string.h> + +#include <linux/rkisp1-config.h> +#include <linux/videodev2.h> + +#include <libcamera/base/log.h> +#include <libcamera/base/utils.h> + +namespace libcamera { + +LOG_DEFINE_CATEGORY(RkISP1Params) + +namespace ipa::rkisp1 { + +namespace { + +struct BlockTypeInfo { + enum rkisp1_ext_params_block_type type; + size_t size; + size_t offset; + uint32_t enableBit; +}; + +#define RKISP1_BLOCK_TYPE_ENTRY(block, id, type, category, bit) \ + { BlockType::block, { \ + RKISP1_EXT_PARAMS_BLOCK_TYPE_##id, \ + sizeof(struct rkisp1_cif_isp_##type##_config), \ + offsetof(struct rkisp1_params_cfg, category.type##_config), \ + RKISP1_CIF_ISP_MODULE_##bit, \ + } } + +#define RKISP1_BLOCK_TYPE_ENTRY_MEAS(block, id, type) \ + RKISP1_BLOCK_TYPE_ENTRY(block, id##_MEAS, type, meas, id) + +#define RKISP1_BLOCK_TYPE_ENTRY_OTHERS(block, id, type) \ + RKISP1_BLOCK_TYPE_ENTRY(block, id, type, others, id) + +#define RKISP1_BLOCK_TYPE_ENTRY_EXT(block, id, type) \ + { BlockType::block, { \ + RKISP1_EXT_PARAMS_BLOCK_TYPE_##id, \ + sizeof(struct rkisp1_cif_isp_##type##_config), \ + 0, 0, \ + } } + +const std::map<BlockType, BlockTypeInfo> kBlockTypeInfo = { + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Bls, BLS, bls), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Dpcc, DPCC, dpcc), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Sdg, SDG, sdg), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(AwbGain, AWB_GAIN, awb_gain), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Flt, FLT, flt), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Bdm, BDM, bdm), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Ctk, CTK, ctk), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Goc, GOC, goc), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Dpf, DPF, dpf), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(DpfStrength, DPF_STRENGTH, dpf_strength), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Cproc, CPROC, cproc), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Ie, IE, ie), + RKISP1_BLOCK_TYPE_ENTRY_OTHERS(Lsc, LSC, lsc), + RKISP1_BLOCK_TYPE_ENTRY_MEAS(Awb, AWB, awb_meas), + RKISP1_BLOCK_TYPE_ENTRY_MEAS(Hst, HST, hst), + RKISP1_BLOCK_TYPE_ENTRY_MEAS(Aec, AEC, aec), + RKISP1_BLOCK_TYPE_ENTRY_MEAS(Afc, AFC, afc), + RKISP1_BLOCK_TYPE_ENTRY_EXT(CompandBls, COMPAND_BLS, compand_bls), + RKISP1_BLOCK_TYPE_ENTRY_EXT(CompandExpand, COMPAND_EXPAND, compand_curve), + RKISP1_BLOCK_TYPE_ENTRY_EXT(CompandCompress, COMPAND_COMPRESS, compand_curve), +}; + +} /* namespace */ + +RkISP1ParamsBlockBase::RkISP1ParamsBlockBase(RkISP1Params *params, BlockType type, + const Span<uint8_t> &data) + : params_(params), type_(type) +{ + if (params_->format() == V4L2_META_FMT_RK_ISP1_EXT_PARAMS) { + header_ = data.subspan(0, sizeof(rkisp1_ext_params_block_header)); + data_ = data.subspan(sizeof(rkisp1_ext_params_block_header)); + } else { + data_ = data; + } +} + +void RkISP1ParamsBlockBase::setEnabled(bool enabled) +{ + /* + * For the legacy fixed format, blocks are enabled in the top-level + * header. Delegate to the RkISP1Params class. + */ + if (params_->format() == V4L2_META_FMT_RK_ISP1_PARAMS) + return params_->setBlockEnabled(type_, enabled); + + /* + * For the extensible format, set the enable and disable flags in the + * block header directly. + */ + struct rkisp1_ext_params_block_header *header = + reinterpret_cast<struct rkisp1_ext_params_block_header *>(header_.data()); + header->flags &= ~(RKISP1_EXT_PARAMS_FL_BLOCK_ENABLE | + RKISP1_EXT_PARAMS_FL_BLOCK_DISABLE); + header->flags |= enabled ? RKISP1_EXT_PARAMS_FL_BLOCK_ENABLE + : RKISP1_EXT_PARAMS_FL_BLOCK_DISABLE; +} + +RkISP1Params::RkISP1Params(uint32_t format, Span<uint8_t> data) + : format_(format), data_(data), used_(0) +{ + if (format_ == V4L2_META_FMT_RK_ISP1_EXT_PARAMS) { + struct rkisp1_ext_params_cfg *cfg = + reinterpret_cast<struct rkisp1_ext_params_cfg *>(data.data()); + + cfg->version = RKISP1_EXT_PARAM_BUFFER_V1; + cfg->data_size = 0; + + used_ += offsetof(struct rkisp1_ext_params_cfg, data); + } else { + memset(data.data(), 0, data.size()); + used_ = sizeof(struct rkisp1_params_cfg); + } +} + +void RkISP1Params::setBlockEnabled(BlockType type, bool enabled) +{ + const BlockTypeInfo &info = kBlockTypeInfo.at(type); + + struct rkisp1_params_cfg *cfg = + reinterpret_cast<struct rkisp1_params_cfg *>(data_.data()); + if (enabled) + cfg->module_ens |= info.enableBit; + else + cfg->module_ens &= ~info.enableBit; +} + +Span<uint8_t> RkISP1Params::block(BlockType type) +{ + auto infoIt = kBlockTypeInfo.find(type); + if (infoIt == kBlockTypeInfo.end()) { + LOG(RkISP1Params, Error) + << "Invalid parameters block type " + << utils::to_underlying(type); + return {}; + } + + const BlockTypeInfo &info = infoIt->second; + + /* + * For the legacy format, return a block referencing the fixed location + * of the data. + */ + if (format_ == V4L2_META_FMT_RK_ISP1_PARAMS) { + /* + * Blocks available only in extended parameters have an offset + * of 0. Return nullptr in that case. + */ + if (info.offset == 0) { + LOG(RkISP1Params, Error) + << "Block type " << utils::to_underlying(type) + << " unavailable in fixed parameters format"; + return {}; + } + + struct rkisp1_params_cfg *cfg = + reinterpret_cast<struct rkisp1_params_cfg *>(data_.data()); + + cfg->module_cfg_update |= info.enableBit; + cfg->module_en_update |= info.enableBit; + + return data_.subspan(info.offset, info.size); + } + + /* + * For the extensible format, allocate memory for the block, including + * the header. Look up the block in the cache first. If an algorithm + * requests the same block type twice, it should get the same block. + */ + auto cacheIt = blocks_.find(type); + if (cacheIt != blocks_.end()) + return cacheIt->second; + + /* Make sure we don't run out of space. */ + size_t size = sizeof(struct rkisp1_ext_params_block_header) + + ((info.size + 7) & ~7); + if (size > data_.size() - used_) { + LOG(RkISP1Params, Error) + << "Out of memory to allocate block type " + << utils::to_underlying(type); + return {}; + } + + /* Allocate a new block, clear its memory, and initialize its header. */ + Span<uint8_t> block = data_.subspan(used_, size); + used_ += size; + + struct rkisp1_ext_params_cfg *cfg = + reinterpret_cast<struct rkisp1_ext_params_cfg *>(data_.data()); + cfg->data_size += size; + + memset(block.data(), 0, block.size()); + + struct rkisp1_ext_params_block_header *header = + reinterpret_cast<struct rkisp1_ext_params_block_header *>(block.data()); + header->type = info.type; + header->size = block.size(); + + /* Update the cache. */ + blocks_[type] = block; + + return block; +} + +} /* namespace ipa::rkisp1 */ + +} /* namespace libcamera */ diff --git a/src/ipa/rkisp1/params.h b/src/ipa/rkisp1/params.h new file mode 100644 index 00000000..40450e34 --- /dev/null +++ b/src/ipa/rkisp1/params.h @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * + * RkISP1 ISP Parameters + */ + +#pragma once + +#include <map> +#include <stdint.h> + +#include <linux/rkisp1-config.h> + +#include <libcamera/base/class.h> +#include <libcamera/base/span.h> + +namespace libcamera { + +namespace ipa::rkisp1 { + +enum class BlockType { + Bls, + Dpcc, + Sdg, + AwbGain, + Flt, + Bdm, + Ctk, + Goc, + Dpf, + DpfStrength, + Cproc, + Ie, + Lsc, + Awb, + Hst, + Aec, + Afc, + CompandBls, + CompandExpand, + CompandCompress, +}; + +namespace details { + +template<BlockType B> +struct block_type { +}; + +#define RKISP1_DEFINE_BLOCK_TYPE(blockType, blockStruct) \ +template<> \ +struct block_type<BlockType::blockType> { \ + using type = struct rkisp1_cif_isp_##blockStruct##_config; \ +}; + +RKISP1_DEFINE_BLOCK_TYPE(Bls, bls) +RKISP1_DEFINE_BLOCK_TYPE(Dpcc, dpcc) +RKISP1_DEFINE_BLOCK_TYPE(Sdg, sdg) +RKISP1_DEFINE_BLOCK_TYPE(AwbGain, awb_gain) +RKISP1_DEFINE_BLOCK_TYPE(Flt, flt) +RKISP1_DEFINE_BLOCK_TYPE(Bdm, bdm) +RKISP1_DEFINE_BLOCK_TYPE(Ctk, ctk) +RKISP1_DEFINE_BLOCK_TYPE(Goc, goc) +RKISP1_DEFINE_BLOCK_TYPE(Dpf, dpf) +RKISP1_DEFINE_BLOCK_TYPE(DpfStrength, dpf_strength) +RKISP1_DEFINE_BLOCK_TYPE(Cproc, cproc) +RKISP1_DEFINE_BLOCK_TYPE(Ie, ie) +RKISP1_DEFINE_BLOCK_TYPE(Lsc, lsc) +RKISP1_DEFINE_BLOCK_TYPE(Awb, awb_meas) +RKISP1_DEFINE_BLOCK_TYPE(Hst, hst) +RKISP1_DEFINE_BLOCK_TYPE(Aec, aec) +RKISP1_DEFINE_BLOCK_TYPE(Afc, afc) +RKISP1_DEFINE_BLOCK_TYPE(CompandBls, compand_bls) +RKISP1_DEFINE_BLOCK_TYPE(CompandExpand, compand_curve) +RKISP1_DEFINE_BLOCK_TYPE(CompandCompress, compand_curve) + +} /* namespace details */ + +class RkISP1Params; + +class RkISP1ParamsBlockBase +{ +public: + RkISP1ParamsBlockBase(RkISP1Params *params, BlockType type, + const Span<uint8_t> &data); + + Span<uint8_t> data() const { return data_; } + + void setEnabled(bool enabled); + +private: + LIBCAMERA_DISABLE_COPY(RkISP1ParamsBlockBase) + + RkISP1Params *params_; + BlockType type_; + Span<uint8_t> header_; + Span<uint8_t> data_; +}; + +template<BlockType B> +class RkISP1ParamsBlock : public RkISP1ParamsBlockBase +{ +public: + using Type = typename details::block_type<B>::type; + + RkISP1ParamsBlock(RkISP1Params *params, const Span<uint8_t> &data) + : RkISP1ParamsBlockBase(params, B, data) + { + } + + const Type *operator->() const + { + return reinterpret_cast<const Type *>(data().data()); + } + + Type *operator->() + { + return reinterpret_cast<Type *>(data().data()); + } + + const Type &operator*() const & + { + return *reinterpret_cast<const Type *>(data().data()); + } + + Type &operator*() & + { + return *reinterpret_cast<Type *>(data().data()); + } +}; + +class RkISP1Params +{ +public: + RkISP1Params(uint32_t format, Span<uint8_t> data); + + template<BlockType B> + RkISP1ParamsBlock<B> block() + { + return RkISP1ParamsBlock<B>(this, block(B)); + } + + uint32_t format() const { return format_; } + size_t size() const { return used_; } + +private: + friend class RkISP1ParamsBlockBase; + + Span<uint8_t> block(BlockType type); + void setBlockEnabled(BlockType type, bool enabled); + + uint32_t format_; + + Span<uint8_t> data_; + size_t used_; + + std::map<BlockType, Span<uint8_t>> blocks_; +}; + +} /* namespace ipa::rkisp1 */ + +} /* namespace libcamera*/ diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index 6544c925..1ed7d7d9 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -2,12 +2,12 @@ /* * Copyright (C) 2019, Google Inc. * - * rkisp1.cpp - RkISP1 Image Processing Algorithms + * RkISP1 Image Processing Algorithms */ #include <algorithm> -#include <math.h> -#include <queue> +#include <array> +#include <chrono> #include <stdint.h> #include <string.h> @@ -18,20 +18,22 @@ #include <libcamera/base/log.h> #include <libcamera/control_ids.h> +#include <libcamera/controls.h> #include <libcamera/framebuffer.h> +#include <libcamera/request.h> + #include <libcamera/ipa/ipa_interface.h> #include <libcamera/ipa/ipa_module_info.h> #include <libcamera/ipa/rkisp1_ipa_interface.h> -#include <libcamera/request.h> #include "libcamera/internal/formats.h" #include "libcamera/internal/mapped_framebuffer.h" #include "libcamera/internal/yaml_parser.h" #include "algorithms/algorithm.h" -#include "libipa/camera_sensor_helper.h" #include "ipa_context.h" +#include "params.h" namespace libcamera { @@ -63,9 +65,9 @@ public: void unmapBuffers(const std::vector<unsigned int> &ids) override; void queueRequest(const uint32_t frame, const ControlList &controls) override; - void fillParamsBuffer(const uint32_t frame, const uint32_t bufferId) override; - void processStatsBuffer(const uint32_t frame, const uint32_t bufferId, - const ControlList &sensorControls) override; + void computeParams(const uint32_t frame, const uint32_t bufferId) override; + void processStats(const uint32_t frame, const uint32_t bufferId, + const ControlList &sensorControls) override; protected: std::string logPrefix() const override; @@ -81,29 +83,39 @@ private: ControlInfoMap sensorControls_; - /* revision-specific data */ - rkisp1_cif_isp_version hwRevision_; - unsigned int hwHistBinNMax_; - unsigned int hwGammaOutMaxSamples_; - unsigned int hwHistogramWeightGridsSize_; - - /* Interface to the Camera Helper */ - std::unique_ptr<CameraSensorHelper> camHelper_; - /* Local parameter storage */ struct IPAContext context_; }; namespace { +const IPAHwSettings ipaHwSettingsV10{ + RKISP1_CIF_ISP_AE_MEAN_MAX_V10, + RKISP1_CIF_ISP_HIST_BIN_N_MAX_V10, + RKISP1_CIF_ISP_HISTOGRAM_WEIGHT_GRIDS_SIZE_V10, + RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES_V10, + false, +}; + +const IPAHwSettings ipaHwSettingsIMX8MP{ + RKISP1_CIF_ISP_AE_MEAN_MAX_V10, + RKISP1_CIF_ISP_HIST_BIN_N_MAX_V10, + RKISP1_CIF_ISP_HISTOGRAM_WEIGHT_GRIDS_SIZE_V10, + RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES_V10, + true, +}; + +const IPAHwSettings ipaHwSettingsV12{ + RKISP1_CIF_ISP_AE_MEAN_MAX_V12, + RKISP1_CIF_ISP_HIST_BIN_N_MAX_V12, + RKISP1_CIF_ISP_HISTOGRAM_WEIGHT_GRIDS_SIZE_V12, + RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES_V12, + false, +}; + /* List of controls handled by the RkISP1 IPA */ const ControlInfoMap::Map rkisp1Controls{ - { &controls::AeEnable, ControlInfo(false, true) }, - { &controls::AwbEnable, ControlInfo(false, true) }, - { &controls::ColourGains, ControlInfo(0.0f, 3.996f, 1.0f) }, - { &controls::Brightness, ControlInfo(-1.0f, 0.993f, 0.0f) }, - { &controls::Contrast, ControlInfo(0.0f, 1.993f, 1.0f) }, - { &controls::Saturation, ControlInfo(0.0f, 1.993f, 1.0f) }, + { &controls::DebugMetadataEnable, ControlInfo(false, true, false) }, { &controls::Sharpness, ControlInfo(0.0f, 10.0f, 1.0f) }, { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) }, }; @@ -111,7 +123,7 @@ const ControlInfoMap::Map rkisp1Controls{ } /* namespace */ IPARkISP1::IPARkISP1() - : context_({ {}, {}, { kMaxFrameContexts } }) + : context_(kMaxFrameContexts) { } @@ -128,14 +140,13 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision, /* \todo Add support for other revisions */ switch (hwRevision) { case RKISP1_V10: - hwHistBinNMax_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V10; - hwGammaOutMaxSamples_ = RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES_V10; - hwHistogramWeightGridsSize_ = RKISP1_CIF_ISP_HISTOGRAM_WEIGHT_GRIDS_SIZE_V10; + context_.hw = &ipaHwSettingsV10; + break; + case RKISP1_V_IMX8MP: + context_.hw = &ipaHwSettingsIMX8MP; break; case RKISP1_V12: - hwHistBinNMax_ = RKISP1_CIF_ISP_HIST_BIN_N_MAX_V12; - hwGammaOutMaxSamples_ = RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES_V12; - hwHistogramWeightGridsSize_ = RKISP1_CIF_ISP_HISTOGRAM_WEIGHT_GRIDS_SIZE_V12; + context_.hw = &ipaHwSettingsV12; break; default: LOG(IPARkISP1, Error) @@ -146,19 +157,18 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision, LOG(IPARkISP1, Debug) << "Hardware revision is " << hwRevision; - /* Cache the value to set it in configure. */ - hwRevision_ = static_cast<rkisp1_cif_isp_version>(hwRevision); + context_.sensorInfo = sensorInfo; - camHelper_ = CameraSensorHelperFactoryBase::create(settings.sensorModel); - if (!camHelper_) { + context_.camHelper = CameraSensorHelperFactoryBase::create(settings.sensorModel); + if (!context_.camHelper) { LOG(IPARkISP1, Error) << "Failed to create camera sensor helper for " << settings.sensorModel; return -ENODEV; } - context_.configuration.sensor.lineDuration = sensorInfo.minLineLength - * 1.0s / sensorInfo.pixelRate; + context_.configuration.sensor.lineDuration = + sensorInfo.minLineLength * 1.0s / sensorInfo.pixelRate; /* Load the tuning data file. */ File file(settings.configurationFile); @@ -199,8 +209,7 @@ int IPARkISP1::init(const IPASettings &settings, unsigned int hwRevision, int IPARkISP1::start() { - setControls(0); - + /* \todo Properly handle startup controls. */ return 0; } @@ -232,8 +241,7 @@ int IPARkISP1::configure(const IPAConfigInfo &ipaConfig, context_.activeState = {}; context_.frameContexts.clear(); - /* Set the hardware revision for the algorithms. */ - context_.configuration.hw.revision = hwRevision_; + context_.configuration.paramFormat = ipaConfig.paramFormat; const IPACameraSensorInfo &info = ipaConfig.sensorInfo; const ControlInfo vBlank = sensorControls_.find(V4L2_CID_VBLANK)->second; @@ -246,17 +254,19 @@ int IPARkISP1::configure(const IPAConfigInfo &ipaConfig, /* * When the AGC computes the new exposure values for a frame, it needs - * to know the limits for shutter speed and analogue gain. - * As it depends on the sensor, update it with the controls. + * to know the limits for exposure time and analogue gain. As it depends + * on the sensor, update it with the controls. * - * \todo take VBLANK into account for maximum shutter speed + * \todo take VBLANK into account for maximum exposure time */ - context_.configuration.sensor.minShutterSpeed = + context_.configuration.sensor.minExposureTime = minExposure * context_.configuration.sensor.lineDuration; - context_.configuration.sensor.maxShutterSpeed = + context_.configuration.sensor.maxExposureTime = maxExposure * context_.configuration.sensor.lineDuration; - context_.configuration.sensor.minAnalogueGain = camHelper_->gain(minGain); - context_.configuration.sensor.maxAnalogueGain = camHelper_->gain(maxGain); + context_.configuration.sensor.minAnalogueGain = + context_.camHelper->gain(minGain); + context_.configuration.sensor.maxAnalogueGain = + context_.camHelper->gain(maxGain); context_.configuration.raw = std::any_of(streamConfig.begin(), streamConfig.end(), [](auto &cfg) -> bool { @@ -314,6 +324,7 @@ void IPARkISP1::unmapBuffers(const std::vector<unsigned int> &ids) void IPARkISP1::queueRequest(const uint32_t frame, const ControlList &controls) { IPAFrameContext &frameContext = context_.frameContexts.alloc(frame); + context_.debugMetadata.enableByControl(controls); for (auto const &a : algorithms()) { Algorithm *algo = static_cast<Algorithm *>(a.get()); @@ -323,25 +334,21 @@ void IPARkISP1::queueRequest(const uint32_t frame, const ControlList &controls) } } -void IPARkISP1::fillParamsBuffer(const uint32_t frame, const uint32_t bufferId) +void IPARkISP1::computeParams(const uint32_t frame, const uint32_t bufferId) { IPAFrameContext &frameContext = context_.frameContexts.get(frame); - rkisp1_params_cfg *params = - reinterpret_cast<rkisp1_params_cfg *>( - mappedBuffers_.at(bufferId).planes()[0].data()); - - /* Prepare parameters buffer. */ - memset(params, 0, sizeof(*params)); + RkISP1Params params(context_.configuration.paramFormat, + mappedBuffers_.at(bufferId).planes()[0]); for (auto const &algo : algorithms()) - algo->prepare(context_, frame, frameContext, params); + algo->prepare(context_, frame, frameContext, ¶ms); - paramsBufferReady.emit(frame); + paramsComputed.emit(frame, params.size()); } -void IPARkISP1::processStatsBuffer(const uint32_t frame, const uint32_t bufferId, - const ControlList &sensorControls) +void IPARkISP1::processStats(const uint32_t frame, const uint32_t bufferId, + const ControlList &sensorControls) { IPAFrameContext &frameContext = context_.frameContexts.get(frame); @@ -357,7 +364,7 @@ void IPARkISP1::processStatsBuffer(const uint32_t frame, const uint32_t bufferId frameContext.sensor.exposure = sensorControls.get(V4L2_CID_EXPOSURE).get<int32_t>(); frameContext.sensor.gain = - camHelper_->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get<int32_t>()); + context_.camHelper->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get<int32_t>()); ControlList metadata(controls::controls); @@ -370,6 +377,7 @@ void IPARkISP1::processStatsBuffer(const uint32_t frame, const uint32_t bufferId setControls(frame); + context_.debugMetadata.moveEntries(metadata); metadataReady.emit(frame, metadata); } @@ -394,9 +402,9 @@ void IPARkISP1::updateControls(const IPACameraSensorInfo &sensorInfo, /* Compute the analogue gain limits. */ const ControlInfo &v4l2Gain = sensorControls.find(V4L2_CID_ANALOGUE_GAIN)->second; - float minGain = camHelper_->gain(v4l2Gain.min().get<int32_t>()); - float maxGain = camHelper_->gain(v4l2Gain.max().get<int32_t>()); - float defGain = camHelper_->gain(v4l2Gain.def().get<int32_t>()); + float minGain = context_.camHelper->gain(v4l2Gain.min().get<int32_t>()); + float maxGain = context_.camHelper->gain(v4l2Gain.max().get<int32_t>()); + float defGain = context_.camHelper->gain(v4l2Gain.def().get<int32_t>()); ctrlMap.emplace(std::piecewise_construct, std::forward_as_tuple(&controls::AnalogueGain), std::forward_as_tuple(minGain, maxGain, defGain)); @@ -424,10 +432,11 @@ void IPARkISP1::updateControls(const IPACameraSensorInfo &sensorInfo, frameDurations[i] = frameSize / (sensorInfo.pixelRate / 1000000U); } - ctrlMap[&controls::FrameDurationLimits] = ControlInfo(frameDurations[0], - frameDurations[1], - frameDurations[2]); + /* \todo Move this (and other agc-related controls) to agc */ + context_.ctrlMap[&controls::FrameDurationLimits] = + ControlInfo(frameDurations[0], frameDurations[1], frameDurations[2]); + ctrlMap.insert(context_.ctrlMap.begin(), context_.ctrlMap.end()); *ipaControls = ControlInfoMap(std::move(ctrlMap), controls::controls); } @@ -440,11 +449,17 @@ void IPARkISP1::setControls(unsigned int frame) IPAFrameContext &frameContext = context_.frameContexts.get(frame); uint32_t exposure = frameContext.agc.exposure; - uint32_t gain = camHelper_->gainCode(frameContext.agc.gain); + uint32_t gain = context_.camHelper->gainCode(frameContext.agc.gain); + uint32_t vblank = frameContext.agc.vblank; + + LOG(IPARkISP1, Debug) + << "Set controls for frame " << frame << ": exposure " << exposure + << ", gain " << frameContext.agc.gain << ", vblank " << vblank; ControlList ctrls(sensorControls_); ctrls.set(V4L2_CID_EXPOSURE, static_cast<int32_t>(exposure)); ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast<int32_t>(gain)); + ctrls.set(V4L2_CID_VBLANK, static_cast<int32_t>(vblank)); setSensorControls.emit(frame, ctrls); } @@ -459,7 +474,7 @@ extern "C" { const struct IPAModuleInfo ipaModuleInfo = { IPA_MODULE_API_VERSION, 1, - "PipelineHandlerRkISP1", + "rkisp1", "rkisp1", }; |