summaryrefslogtreecommitdiff
path: root/src/ipa/ipu3
diff options
context:
space:
mode:
Diffstat (limited to 'src/ipa/ipu3')
-rw-r--r--src/ipa/ipu3/algorithms/af.cpp5
-rw-r--r--src/ipa/ipu3/algorithms/af.h2
-rw-r--r--src/ipa/ipu3/algorithms/agc.cpp310
-rw-r--r--src/ipa/ipu3/algorithms/agc.h33
-rw-r--r--src/ipa/ipu3/algorithms/algorithm.h2
-rw-r--r--src/ipa/ipu3/algorithms/awb.cpp71
-rw-r--r--src/ipa/ipu3/algorithms/awb.h21
-rw-r--r--src/ipa/ipu3/algorithms/blc.cpp8
-rw-r--r--src/ipa/ipu3/algorithms/blc.h2
-rw-r--r--src/ipa/ipu3/algorithms/tone_mapping.cpp2
-rw-r--r--src/ipa/ipu3/algorithms/tone_mapping.h2
-rw-r--r--src/ipa/ipu3/data/meson.build3
-rw-r--r--src/ipa/ipu3/ipa_context.cpp17
-rw-r--r--src/ipa/ipu3/ipa_context.h16
-rw-r--r--src/ipa/ipu3/ipu3-ipa-design-guide.rst14
-rw-r--r--src/ipa/ipu3/ipu3.cpp61
-rw-r--r--src/ipa/ipu3/meson.build8
-rw-r--r--src/ipa/ipu3/module.h2
18 files changed, 209 insertions, 370 deletions
diff --git a/src/ipa/ipu3/algorithms/af.cpp b/src/ipa/ipu3/algorithms/af.cpp
index 12927eec..cf68fb59 100644
--- a/src/ipa/ipu3/algorithms/af.cpp
+++ b/src/ipa/ipu3/algorithms/af.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Red Hat
*
- * af.cpp - IPU3 auto focus algorithm
+ * IPU3 auto focus algorithm
*/
#include "af.h"
@@ -11,7 +11,6 @@
#include <chrono>
#include <cmath>
#include <fcntl.h>
-#include <numeric>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
@@ -23,8 +22,6 @@
#include <libcamera/ipa/core_ipa_interface.h>
-#include "libipa/histogram.h"
-
/**
* \file af.h
*/
diff --git a/src/ipa/ipu3/algorithms/af.h b/src/ipa/ipu3/algorithms/af.h
index c6168e30..68126d46 100644
--- a/src/ipa/ipu3/algorithms/af.h
+++ b/src/ipa/ipu3/algorithms/af.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Red Hat
*
- * af.h - IPU3 Af algorithm
+ * IPU3 Af algorithm
*/
#pragma once
diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp
index 606a237a..39d0aebb 100644
--- a/src/ipa/ipu3/algorithms/agc.cpp
+++ b/src/ipa/ipu3/algorithms/agc.cpp
@@ -2,21 +2,22 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * ipu3_agc.cpp - AGC/AEC mean-based control algorithm
+ * AGC/AEC mean-based control algorithm
*/
#include "agc.h"
#include <algorithm>
#include <chrono>
-#include <cmath>
#include <libcamera/base/log.h>
#include <libcamera/base/utils.h>
#include <libcamera/control_ids.h>
+
#include <libcamera/ipa/core_ipa_interface.h>
+#include "libipa/colours.h"
#include "libipa/histogram.h"
/**
@@ -33,7 +34,7 @@ namespace ipa::ipu3::algorithms {
* \class Agc
* \brief A mean-based auto-exposure algorithm
*
- * This algorithm calculates a shutter time and an analogue gain so that the
+ * This algorithm calculates an exposure time and an analogue gain so that the
* average value of the green channel of the brightest 2% of pixels approaches
* 0.5. The AWB gains are not used here, and all cells in the grid have the same
* weight, like an average-metering case. In this metering mode, the camera uses
@@ -51,29 +52,37 @@ LOG_DEFINE_CATEGORY(IPU3Agc)
static constexpr double kMinAnalogueGain = 1.0;
/* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */
-static constexpr utils::Duration kMaxShutterSpeed = 60ms;
+static constexpr utils::Duration kMaxExposureTime = 60ms;
/* Histogram constants */
static constexpr uint32_t knumHistogramBins = 256;
-/* Target value to reach for the top 2% of the histogram */
-static constexpr double kEvGainTarget = 0.5;
-
-/* Number of frames to wait before calculating stats on minimum exposure */
-static constexpr uint32_t kNumStartupFrames = 10;
+Agc::Agc()
+ : minExposureTime_(0s), maxExposureTime_(0s)
+{
+}
-/*
- * Relative luminance target.
+/**
+ * \brief Initialise the AGC algorithm from tuning files
+ * \param[in] context The shared IPA context
+ * \param[in] tuningData The YamlObject containing Agc tuning data
+ *
+ * This function calls the base class' tuningData parsers to discover which
+ * control values are supported.
*
- * It's a number that's chosen so that, when the camera points at a grey
- * target, the resulting image brightness is considered right.
+ * \return 0 on success or errors from the base class
*/
-static constexpr double kRelativeLuminanceTarget = 0.16;
-
-Agc::Agc()
- : frameCount_(0), minShutterSpeed_(0s),
- maxShutterSpeed_(0s), filteredExposure_(0s)
+int Agc::init(IPAContext &context, const YamlObject &tuningData)
{
+ int ret;
+
+ ret = parseTuningData(tuningData);
+ if (ret)
+ return ret;
+
+ context.ctrlMap.merge(controls());
+
+ return 0;
}
/**
@@ -90,10 +99,11 @@ int Agc::configure(IPAContext &context,
IPAActiveState &activeState = context.activeState;
stride_ = configuration.grid.stride;
+ bdsGrid_ = configuration.grid.bdsGrid;
- minShutterSpeed_ = configuration.agc.minShutterSpeed;
- maxShutterSpeed_ = std::min(configuration.agc.maxShutterSpeed,
- kMaxShutterSpeed);
+ minExposureTime_ = configuration.agc.minExposureTime;
+ maxExposureTime_ = std::min(configuration.agc.maxExposureTime,
+ kMaxExposureTime);
minAnalogueGain_ = std::max(configuration.agc.minAnalogueGain, kMinAnalogueGain);
maxAnalogueGain_ = configuration.agc.maxAnalogueGain;
@@ -102,168 +112,53 @@ int Agc::configure(IPAContext &context,
activeState.agc.gain = minAnalogueGain_;
activeState.agc.exposure = 10ms / configuration.sensor.lineDuration;
- frameCount_ = 0;
+ context.activeState.agc.constraintMode = constraintModes().begin()->first;
+ context.activeState.agc.exposureMode = exposureModeHelpers().begin()->first;
+
+ /* \todo Run this again when FrameDurationLimits is passed in */
+ setLimits(minExposureTime_, maxExposureTime_, minAnalogueGain_,
+ maxAnalogueGain_);
+ resetFrameCount();
+
return 0;
}
-/**
- * \brief Estimate the mean value of the top 2% of the histogram
- * \param[in] stats The statistics computed by the ImgU
- * \param[in] grid The grid used to store the statistics in the IPU3
- * \return The mean value of the top 2% of the histogram
- */
-double Agc::measureBrightness(const ipu3_uapi_stats_3a *stats,
- const ipu3_uapi_grid_config &grid) const
+Histogram Agc::parseStatistics(const ipu3_uapi_stats_3a *stats,
+ const ipu3_uapi_grid_config &grid)
{
- /* Initialise the histogram array */
uint32_t hist[knumHistogramBins] = { 0 };
+ rgbTriples_.clear();
+
for (unsigned int cellY = 0; cellY < grid.height; cellY++) {
for (unsigned int cellX = 0; cellX < grid.width; cellX++) {
uint32_t cellPosition = cellY * stride_ + cellX;
const ipu3_uapi_awb_set_item *cell =
reinterpret_cast<const ipu3_uapi_awb_set_item *>(
- &stats->awb_raw_buffer.meta_data[cellPosition]
- );
+ &stats->awb_raw_buffer.meta_data[cellPosition]);
+
+ rgbTriples_.push_back({
+ cell->R_avg,
+ (cell->Gr_avg + cell->Gb_avg) / 2,
+ cell->B_avg
+ });
- uint8_t gr = cell->Gr_avg;
- uint8_t gb = cell->Gb_avg;
/*
* Store the average green value to estimate the
* brightness. Even the overexposed pixels are
* taken into account.
*/
- hist[(gr + gb) / 2]++;
+ hist[(cell->Gr_avg + cell->Gb_avg) / 2]++;
}
}
- /* Estimate the quantile mean of the top 2% of the histogram. */
- return Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0);
-}
-
-/**
- * \brief Apply a filter on the exposure value to limit the speed of changes
- * \param[in] exposureValue The target exposure from the AGC algorithm
- *
- * The speed of the filter is adaptive, and will produce the target quicker
- * during startup, or when the target exposure is within 20% of the most recent
- * filter output.
- *
- * \return The filtered exposure
- */
-utils::Duration Agc::filterExposure(utils::Duration exposureValue)
-{
- double speed = 0.2;
-
- /* Adapt instantly if we are in startup phase. */
- if (frameCount_ < kNumStartupFrames)
- speed = 1.0;
-
- /*
- * If we are close to the desired result, go faster to avoid making
- * multiple micro-adjustments.
- * \todo Make this customisable?
- */
- if (filteredExposure_ < 1.2 * exposureValue &&
- filteredExposure_ > 0.8 * exposureValue)
- speed = sqrt(speed);
-
- filteredExposure_ = speed * exposureValue +
- filteredExposure_ * (1.0 - speed);
-
- LOG(IPU3Agc, Debug) << "After filtering, exposure " << filteredExposure_;
-
- return filteredExposure_;
-}
-
-/**
- * \brief Estimate the new exposure and gain values
- * \param[inout] frameContext The shared IPA frame Context
- * \param[in] yGain The gain calculated based on the relative luminance target
- * \param[in] iqMeanGain The gain calculated based on the relative luminance target
- */
-void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext,
- double yGain, double iqMeanGain)
-{
- const IPASessionConfiguration &configuration = context.configuration;
- /* Get the effective exposure and gain applied on the sensor. */
- uint32_t exposure = frameContext.sensor.exposure;
- double analogueGain = frameContext.sensor.gain;
-
- /* Use the highest of the two gain estimates. */
- double evGain = std::max(yGain, iqMeanGain);
-
- /* Consider within 1% of the target as correctly exposed */
- if (utils::abs_diff(evGain, 1.0) < 0.01)
- LOG(IPU3Agc, Debug) << "We are well exposed (evGain = "
- << evGain << ")";
-
- /* extracted from Rpi::Agc::computeTargetExposure */
-
- /* Calculate the shutter time in seconds */
- utils::Duration currentShutter = exposure * configuration.sensor.lineDuration;
-
- /*
- * Update the exposure value for the next computation using the values
- * of exposure and gain really used by the sensor.
- */
- utils::Duration effectiveExposureValue = currentShutter * analogueGain;
-
- LOG(IPU3Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain
- << " Shutter speed " << currentShutter
- << " Gain " << analogueGain
- << " Needed ev gain " << evGain;
-
- /*
- * Calculate the current exposure value for the scene as the latest
- * exposure value applied multiplied by the new estimated gain.
- */
- utils::Duration exposureValue = effectiveExposureValue * evGain;
-
- /* Clamp the exposure value to the min and max authorized */
- utils::Duration maxTotalExposure = maxShutterSpeed_ * maxAnalogueGain_;
- exposureValue = std::min(exposureValue, maxTotalExposure);
- LOG(IPU3Agc, Debug) << "Target total exposure " << exposureValue
- << ", maximum is " << maxTotalExposure;
-
- /*
- * Filter the exposure.
- * \todo estimate if we need to desaturate
- */
- exposureValue = filterExposure(exposureValue);
-
- /*
- * Divide the exposure value as new exposure and gain values.
- *
- * Push the shutter time up to the maximum first, and only then
- * increase the gain.
- */
- utils::Duration shutterTime =
- std::clamp<utils::Duration>(exposureValue / minAnalogueGain_,
- minShutterSpeed_, maxShutterSpeed_);
- double stepGain = std::clamp(exposureValue / shutterTime,
- minAnalogueGain_, maxAnalogueGain_);
- LOG(IPU3Agc, Debug) << "Divided up shutter and gain are "
- << shutterTime << " and "
- << stepGain;
-
- IPAActiveState &activeState = context.activeState;
- /* Update the estimated exposure and gain. */
- activeState.agc.exposure = shutterTime / configuration.sensor.lineDuration;
- activeState.agc.gain = stepGain;
+ return Histogram(Span<uint32_t>(hist));
}
/**
* \brief Estimate the relative luminance of the frame with a given gain
- * \param[in] frameContext The shared IPA frame context
- * \param[in] grid The grid used to store the statistics in the IPU3
- * \param[in] stats The IPU3 statistics and ISP results
- * \param[in] gain The gain to apply to the frame
- * \return The relative luminance
- *
- * This function estimates the average relative luminance of the frame that
- * would be output by the sensor if an additional \a gain was applied.
+ * \param[in] gain The gain to apply in estimating luminance
*
* The estimation is based on the AWB statistics for the current frame. Red,
* green and blue averages for all cells are first multiplied by the gain, and
@@ -278,40 +173,22 @@ void Agc::computeExposure(IPAContext &context, IPAFrameContext &frameContext,
*
* More detailed information can be found in:
* https://en.wikipedia.org/wiki/Relative_luminance
+ *
+ * \return The relative luminance of the frame
*/
-double Agc::estimateLuminance(IPAActiveState &activeState,
- const ipu3_uapi_grid_config &grid,
- const ipu3_uapi_stats_3a *stats,
- double gain)
+double Agc::estimateLuminance(double gain) const
{
- double redSum = 0, greenSum = 0, blueSum = 0;
-
- /* Sum the per-channel averages, saturated to 255. */
- for (unsigned int cellY = 0; cellY < grid.height; cellY++) {
- for (unsigned int cellX = 0; cellX < grid.width; cellX++) {
- uint32_t cellPosition = cellY * stride_ + cellX;
-
- const ipu3_uapi_awb_set_item *cell =
- reinterpret_cast<const ipu3_uapi_awb_set_item *>(
- &stats->awb_raw_buffer.meta_data[cellPosition]
- );
- const uint8_t G_avg = (cell->Gr_avg + cell->Gb_avg) / 2;
+ RGB<double> sum{ 0.0 };
- redSum += std::min(cell->R_avg * gain, 255.0);
- greenSum += std::min(G_avg * gain, 255.0);
- blueSum += std::min(cell->B_avg * gain, 255.0);
- }
+ for (unsigned int i = 0; i < rgbTriples_.size(); i++) {
+ sum.r() += std::min(std::get<0>(rgbTriples_[i]) * gain, 255.0);
+ sum.g() += std::min(std::get<1>(rgbTriples_[i]) * gain, 255.0);
+ sum.b() += std::min(std::get<2>(rgbTriples_[i]) * gain, 255.0);
}
- /*
- * Apply the AWB gains to approximate colours correctly, use the Rec.
- * 601 formula to calculate the relative luminance, and normalize it.
- */
- double ySum = redSum * activeState.awb.gains.red * 0.299
- + greenSum * activeState.awb.gains.green * 0.587
- + blueSum * activeState.awb.gains.blue * 0.114;
-
- return ySum / (grid.height * grid.width) / 255;
+ RGB<double> gains{{ rGain_, gGain_, bGain_ }};
+ double ySum = rec601LuminanceFromRGB(sum * gains);
+ return ySum / (bdsGrid_.height * bdsGrid_.width) / 255;
}
/**
@@ -330,44 +207,36 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
const ipu3_uapi_stats_3a *stats,
ControlList &metadata)
{
- /*
- * Estimate the gain needed to have the proportion of pixels in a given
- * desired range. iqMean is the mean value of the top 2% of the
- * cumulative histogram, and we want it to be as close as possible to a
- * configured target.
- */
- double iqMean = measureBrightness(stats, context.configuration.grid.bdsGrid);
- double iqMeanGain = kEvGainTarget * knumHistogramBins / iqMean;
+ Histogram hist = parseStatistics(stats, context.configuration.grid.bdsGrid);
+ rGain_ = context.activeState.awb.gains.red;
+ gGain_ = context.activeState.awb.gains.blue;
+ bGain_ = context.activeState.awb.gains.green;
/*
- * Estimate the gain needed to achieve a relative luminance target. To
- * account for non-linearity caused by saturation, the value needs to be
- * estimated in an iterative process, as multiplying by a gain will not
- * increase the relative luminance by the same factor if some image
- * regions are saturated.
+ * The Agc algorithm needs to know the effective exposure value that was
+ * applied to the sensor when the statistics were collected.
*/
- double yGain = 1.0;
- double yTarget = kRelativeLuminanceTarget;
-
- for (unsigned int i = 0; i < 8; i++) {
- double yValue = estimateLuminance(context.activeState,
- context.configuration.grid.bdsGrid,
- stats, yGain);
- double extraGain = std::min(10.0, yTarget / (yValue + .001));
-
- yGain *= extraGain;
- LOG(IPU3Agc, Debug) << "Y value: " << yValue
- << ", Y target: " << yTarget
- << ", gives gain " << yGain;
- if (extraGain < 1.01)
- break;
- }
-
- computeExposure(context, frameContext, yGain, iqMeanGain);
- frameCount_++;
-
utils::Duration exposureTime = context.configuration.sensor.lineDuration
* frameContext.sensor.exposure;
+ double analogueGain = frameContext.sensor.gain;
+ utils::Duration effectiveExposureValue = exposureTime * analogueGain;
+
+ utils::Duration newExposureTime;
+ double aGain, dGain;
+ std::tie(newExposureTime, aGain, dGain) =
+ calculateNewEv(context.activeState.agc.constraintMode,
+ context.activeState.agc.exposureMode, hist,
+ effectiveExposureValue);
+
+ LOG(IPU3Agc, Debug)
+ << "Divided up exposure time, analogue gain and digital gain are "
+ << newExposureTime << ", " << aGain << " and " << dGain;
+
+ IPAActiveState &activeState = context.activeState;
+ /* Update the estimated exposure time and gain. */
+ activeState.agc.exposure = newExposureTime / context.configuration.sensor.lineDuration;
+ activeState.agc.gain = aGain;
+
metadata.set(controls::AnalogueGain, frameContext.sensor.gain);
metadata.set(controls::ExposureTime, exposureTime.get<std::micro>());
@@ -377,7 +246,6 @@ void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
utils::Duration frameDuration = context.configuration.sensor.lineDuration
* vTotal;
metadata.set(controls::FrameDuration, frameDuration.get<std::micro>());
-
}
REGISTER_IPA_ALGORITHM(Agc, "Agc")
diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h
index 9d6e3ff1..890c271b 100644
--- a/src/ipa/ipu3/algorithms/agc.h
+++ b/src/ipa/ipu3/algorithms/agc.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * agc.h - IPU3 AGC/AEC mean-based control algorithm
+ * IPU3 AGC/AEC mean-based control algorithm
*/
#pragma once
@@ -13,6 +13,9 @@
#include <libcamera/geometry.h>
+#include "libipa/agc_mean_luminance.h"
+#include "libipa/histogram.h"
+
#include "algorithm.h"
namespace libcamera {
@@ -21,12 +24,13 @@ struct IPACameraSensorInfo;
namespace ipa::ipu3::algorithms {
-class Agc : public Algorithm
+class Agc : public Algorithm, public AgcMeanLuminance
{
public:
Agc();
~Agc() = default;
+ int init(IPAContext &context, const YamlObject &tuningData) override;
int configure(IPAContext &context, const IPAConfigInfo &configInfo) override;
void process(IPAContext &context, const uint32_t frame,
IPAFrameContext &frameContext,
@@ -34,27 +38,22 @@ public:
ControlList &metadata) override;
private:
- double measureBrightness(const ipu3_uapi_stats_3a *stats,
- const ipu3_uapi_grid_config &grid) const;
- utils::Duration filterExposure(utils::Duration currentExposure);
- void computeExposure(IPAContext &context, IPAFrameContext &frameContext,
- double yGain, double iqMeanGain);
- double estimateLuminance(IPAActiveState &activeState,
- const ipu3_uapi_grid_config &grid,
- const ipu3_uapi_stats_3a *stats,
- double gain);
-
- uint64_t frameCount_;
+ double estimateLuminance(double gain) const override;
+ Histogram parseStatistics(const ipu3_uapi_stats_3a *stats,
+ const ipu3_uapi_grid_config &grid);
- utils::Duration minShutterSpeed_;
- utils::Duration maxShutterSpeed_;
+ utils::Duration minExposureTime_;
+ utils::Duration maxExposureTime_;
double minAnalogueGain_;
double maxAnalogueGain_;
- utils::Duration filteredExposure_;
-
uint32_t stride_;
+ double rGain_;
+ double gGain_;
+ double bGain_;
+ ipu3_uapi_grid_config bdsGrid_;
+ std::vector<std::tuple<uint8_t, uint8_t, uint8_t>> rgbTriples_;
};
} /* namespace ipa::ipu3::algorithms */
diff --git a/src/ipa/ipu3/algorithms/algorithm.h b/src/ipa/ipu3/algorithms/algorithm.h
index ae134a94..c7801f93 100644
--- a/src/ipa/ipu3/algorithms/algorithm.h
+++ b/src/ipa/ipu3/algorithms/algorithm.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * algorithm.h - IPU3 control algorithm interface
+ * IPU3 control algorithm interface
*/
#pragma once
diff --git a/src/ipa/ipu3/algorithms/awb.cpp b/src/ipa/ipu3/algorithms/awb.cpp
index 5abd4621..55de05d9 100644
--- a/src/ipa/ipu3/algorithms/awb.cpp
+++ b/src/ipa/ipu3/algorithms/awb.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * awb.cpp - AWB control algorithm
+ * AWB control algorithm
*/
#include "awb.h"
@@ -13,6 +13,8 @@
#include <libcamera/control_ids.h>
+#include "libipa/colours.h"
+
/**
* \file awb.h
*/
@@ -301,51 +303,24 @@ void Awb::prepare(IPAContext &context,
params->use.acc_ccm = 1;
}
-/**
- * The function estimates the correlated color temperature using
- * from RGB color space input.
- * In physics and color science, the Planckian locus or black body locus is
- * the path or locus that the color of an incandescent black body would take
- * in a particular chromaticity space as the blackbody temperature changes.
- *
- * If a narrow range of color temperatures is considered (those encapsulating
- * daylight being the most practical case) one can approximate the Planckian
- * locus in order to calculate the CCT in terms of chromaticity coordinates.
- *
- * More detailed information can be found in:
- * https://en.wikipedia.org/wiki/Color_temperature#Approximation
- */
-uint32_t Awb::estimateCCT(double red, double green, double blue)
-{
- /* Convert the RGB values to CIE tristimulus values (XYZ) */
- double X = (-0.14282) * (red) + (1.54924) * (green) + (-0.95641) * (blue);
- double Y = (-0.32466) * (red) + (1.57837) * (green) + (-0.73191) * (blue);
- double Z = (-0.68202) * (red) + (0.77073) * (green) + (0.56332) * (blue);
-
- /* Calculate the normalized chromaticity values */
- double x = X / (X + Y + Z);
- double y = Y / (X + Y + Z);
-
- /* Calculate CCT */
- double n = (x - 0.3320) / (0.1858 - y);
- return 449 * n * n * n + 3525 * n * n + 6823.3 * n + 5520.33;
-}
-
/* Generate an RGB vector with the average values for each zone */
void Awb::generateZones()
{
zones_.clear();
for (unsigned int i = 0; i < kAwbStatsSizeX * kAwbStatsSizeY; i++) {
- RGB zone;
double counted = awbStats_[i].counted;
if (counted >= cellsPerZoneThreshold_) {
- zone.G = awbStats_[i].sum.green / counted;
- if (zone.G >= kMinGreenLevelInZone) {
- zone.R = awbStats_[i].sum.red / counted;
- zone.B = awbStats_[i].sum.blue / counted;
+ RGB<double> zone{{
+ static_cast<double>(awbStats_[i].sum.red),
+ static_cast<double>(awbStats_[i].sum.green),
+ static_cast<double>(awbStats_[i].sum.blue)
+ }};
+
+ zone /= counted;
+
+ if (zone.g() >= kMinGreenLevelInZone)
zones_.push_back(zone);
- }
}
}
}
@@ -412,32 +387,32 @@ void Awb::awbGreyWorld()
* consider some variations, such as normalising all the zones first, or
* doing an L2 average etc.
*/
- std::vector<RGB> &redDerivative(zones_);
- std::vector<RGB> blueDerivative(redDerivative);
+ std::vector<RGB<double>> &redDerivative(zones_);
+ std::vector<RGB<double>> blueDerivative(redDerivative);
std::sort(redDerivative.begin(), redDerivative.end(),
- [](RGB const &a, RGB const &b) {
- return a.G * b.R < b.G * a.R;
+ [](RGB<double> const &a, RGB<double> const &b) {
+ return a.g() * b.r() < b.g() * a.r();
});
std::sort(blueDerivative.begin(), blueDerivative.end(),
- [](RGB const &a, RGB const &b) {
- return a.G * b.B < b.G * a.B;
+ [](RGB<double> const &a, RGB<double> const &b) {
+ return a.g() * b.b() < b.g() * a.b();
});
/* Average the middle half of the values. */
int discard = redDerivative.size() / 4;
- RGB sumRed(0, 0, 0);
- RGB sumBlue(0, 0, 0);
+ RGB<double> sumRed{ 0.0 };
+ RGB<double> sumBlue{ 0.0 };
for (auto ri = redDerivative.begin() + discard,
bi = blueDerivative.begin() + discard;
ri != redDerivative.end() - discard; ri++, bi++)
sumRed += *ri, sumBlue += *bi;
- double redGain = sumRed.G / (sumRed.R + 1),
- blueGain = sumBlue.G / (sumBlue.B + 1);
+ double redGain = sumRed.g() / (sumRed.r() + 1),
+ blueGain = sumBlue.g() / (sumBlue.b() + 1);
/* Color temperature is not relevant in Grey world but still useful to estimate it :-) */
- asyncResults_.temperatureK = estimateCCT(sumRed.R, sumRed.G, sumBlue.B);
+ asyncResults_.temperatureK = estimateCCT({{ sumRed.r(), sumRed.g(), sumBlue.b() }});
/*
* Gain values are unsigned integer value ranging [0, 8) with 13 bit
diff --git a/src/ipa/ipu3/algorithms/awb.h b/src/ipa/ipu3/algorithms/awb.h
index 7a70854e..dbf69c90 100644
--- a/src/ipa/ipu3/algorithms/awb.h
+++ b/src/ipa/ipu3/algorithms/awb.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * awb.h - IPU3 AWB control algorithm
+ * IPU3 AWB control algorithm
*/
#pragma once
@@ -13,6 +13,8 @@
#include <libcamera/geometry.h>
+#include "libcamera/internal/vector.h"
+
#include "algorithm.h"
namespace libcamera {
@@ -48,20 +50,6 @@ public:
ControlList &metadata) override;
private:
- /* \todo Make these structs available to all the ISPs ? */
- struct RGB {
- RGB(double _R = 0, double _G = 0, double _B = 0)
- : R(_R), G(_G), B(_B)
- {
- }
- double R, G, B;
- RGB &operator+=(RGB const &other)
- {
- R += other.R, G += other.G, B += other.B;
- return *this;
- }
- };
-
struct AwbStatus {
double temperatureK;
double redGain;
@@ -75,11 +63,10 @@ private:
void generateAwbStats(const ipu3_uapi_stats_3a *stats);
void clearAwbStats();
void awbGreyWorld();
- uint32_t estimateCCT(double red, double green, double blue);
static constexpr uint16_t threshold(float value);
static constexpr uint16_t gainValue(double gain);
- std::vector<RGB> zones_;
+ std::vector<RGB<double>> zones_;
Accumulator awbStats_[kAwbStatsSizeX * kAwbStatsSizeY];
AwbStatus asyncResults_;
diff --git a/src/ipa/ipu3/algorithms/blc.cpp b/src/ipa/ipu3/algorithms/blc.cpp
index e838072a..35748fb2 100644
--- a/src/ipa/ipu3/algorithms/blc.cpp
+++ b/src/ipa/ipu3/algorithms/blc.cpp
@@ -2,13 +2,11 @@
/*
* Copyright (C) 2021, Google inc.
*
- * blc.cpp - IPU3 Black Level Correction control
+ * IPU3 Black Level Correction control
*/
#include "blc.h"
-#include <string.h>
-
/**
* \file blc.h
* \brief IPU3 Black Level Correction control
@@ -57,8 +55,8 @@ void BlackLevelCorrection::prepare([[maybe_unused]] IPAContext &context,
* tuning processes. This is a first rough approximation.
*/
params->obgrid_param.gr = 64;
- params->obgrid_param.r = 64;
- params->obgrid_param.b = 64;
+ params->obgrid_param.r = 64;
+ params->obgrid_param.b = 64;
params->obgrid_param.gb = 64;
/* Enable the custom black level correction processing */
diff --git a/src/ipa/ipu3/algorithms/blc.h b/src/ipa/ipu3/algorithms/blc.h
index 292bf67b..62748045 100644
--- a/src/ipa/ipu3/algorithms/blc.h
+++ b/src/ipa/ipu3/algorithms/blc.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google inc.
*
- * black_correction.h - IPU3 Black Level Correction control
+ * IPU3 Black Level Correction control
*/
#pragma once
diff --git a/src/ipa/ipu3/algorithms/tone_mapping.cpp b/src/ipa/ipu3/algorithms/tone_mapping.cpp
index a169894c..160338c1 100644
--- a/src/ipa/ipu3/algorithms/tone_mapping.cpp
+++ b/src/ipa/ipu3/algorithms/tone_mapping.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google inc.
*
- * tone_mapping.cpp - IPU3 ToneMapping and Gamma control
+ * IPU3 ToneMapping and Gamma control
*/
#include "tone_mapping.h"
diff --git a/src/ipa/ipu3/algorithms/tone_mapping.h b/src/ipa/ipu3/algorithms/tone_mapping.h
index 5ae35da5..b2b38010 100644
--- a/src/ipa/ipu3/algorithms/tone_mapping.h
+++ b/src/ipa/ipu3/algorithms/tone_mapping.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google inc.
*
- * tone_mapping.h - IPU3 ToneMapping and Gamma control
+ * IPU3 ToneMapping and Gamma control
*/
#pragma once
diff --git a/src/ipa/ipu3/data/meson.build b/src/ipa/ipu3/data/meson.build
index 1f50b630..0f7cd5c6 100644
--- a/src/ipa/ipu3/data/meson.build
+++ b/src/ipa/ipu3/data/meson.build
@@ -5,4 +5,5 @@ conf_files = files([
])
install_data(conf_files,
- install_dir : ipa_data_dir / 'ipu3')
+ install_dir : ipa_data_dir / 'ipu3',
+ install_tag : 'runtime')
diff --git a/src/ipa/ipu3/ipa_context.cpp b/src/ipa/ipu3/ipa_context.cpp
index 959f314f..3b22f791 100644
--- a/src/ipa/ipu3/ipa_context.cpp
+++ b/src/ipa/ipu3/ipa_context.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google Inc.
*
- * ipa_context.cpp - IPU3 IPA Context
+ * IPU3 IPA Context
*/
#include "ipa_context.h"
@@ -39,6 +39,10 @@ namespace libcamera::ipa::ipu3 {
* \struct IPAContext
* \brief Global IPA context data shared between all algorithms
*
+ * \fn IPAContext::IPAContext
+ * \brief Initialize the instance with the given number of frame contexts
+ * \param[in] frameContextSize Size of the frame context ring buffer
+ *
* \var IPAContext::configuration
* \brief The IPA session configuration, immutable during the session
*
@@ -47,6 +51,9 @@ namespace libcamera::ipa::ipu3 {
*
* \var IPAContext::activeState
* \brief The current state of IPA algorithms
+ *
+ * \var IPAContext::ctrlMap
+ * \brief A ControlInfoMap::Map of controls populated by the algorithms
*/
/**
@@ -89,11 +96,11 @@ namespace libcamera::ipa::ipu3 {
* \var IPASessionConfiguration::agc
* \brief AGC parameters configuration of the IPA
*
- * \var IPASessionConfiguration::agc.minShutterSpeed
- * \brief Minimum shutter speed supported with the configured sensor
+ * \var IPASessionConfiguration::agc.minExposureTime
+ * \brief Minimum exposure time supported with the configured sensor
*
- * \var IPASessionConfiguration::agc.maxShutterSpeed
- * \brief Maximum shutter speed supported with the configured sensor
+ * \var IPASessionConfiguration::agc.maxExposureTime
+ * \brief Maximum exposure time supported with the configured sensor
*
* \var IPASessionConfiguration::agc.minAnalogueGain
* \brief Minimum analogue gain supported with the configured sensor
diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h
index e9a3863b..97fcf06c 100644
--- a/src/ipa/ipu3/ipa_context.h
+++ b/src/ipa/ipu3/ipa_context.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google Inc.
*
- * ipa_context.h - IPU3 IPA Context
+ * IPU3 IPA Context
*
*/
@@ -12,6 +12,7 @@
#include <libcamera/base/utils.h>
+#include <libcamera/controls.h>
#include <libcamera/geometry.h>
#include <libipa/fc_queue.h>
@@ -32,8 +33,8 @@ struct IPASessionConfiguration {
} af;
struct {
- utils::Duration minShutterSpeed;
- utils::Duration maxShutterSpeed;
+ utils::Duration minExposureTime;
+ utils::Duration maxExposureTime;
double minAnalogueGain;
double maxAnalogueGain;
} agc;
@@ -55,6 +56,8 @@ struct IPAActiveState {
struct {
uint32_t exposure;
double gain;
+ uint32_t constraintMode;
+ uint32_t exposureMode;
} agc;
struct {
@@ -81,10 +84,17 @@ struct IPAFrameContext : public FrameContext {
};
struct IPAContext {
+ IPAContext(unsigned int frameContextSize)
+ : frameContexts(frameContextSize)
+ {
+ }
+
IPASessionConfiguration configuration;
IPAActiveState activeState;
FCQueue<IPAFrameContext> frameContexts;
+
+ ControlInfoMap::Map ctrlMap;
};
} /* namespace ipa::ipu3 */
diff --git a/src/ipa/ipu3/ipu3-ipa-design-guide.rst b/src/ipa/ipu3/ipu3-ipa-design-guide.rst
index 72506397..85d735c6 100644
--- a/src/ipa/ipu3/ipu3-ipa-design-guide.rst
+++ b/src/ipa/ipu3/ipu3-ipa-design-guide.rst
@@ -27,8 +27,8 @@ from applications, and managing events from the pipeline handler.
└─┬───┬───┬──────┬────┬────┬────┬─┴────▼─┬──┘ 1: init()
│ │ │ │ ▲ │ ▲ │ ▲ │ ▲ │ 2: configure()
│1 │2 │3 │4│ │4│ │4│ │4│ │5 3: mapBuffers(), start()
- │ │ │ │ │ │ │ │ │ │ │ │ 4: (▼) queueRequest(), fillParamsBuffer(), processStatsBuffer()
- ▼ ▼ ▼ ▼ │ ▼ │ ▼ │ ▼ │ ▼ (▲) setSensorControls, paramsBufferReady, metadataReady Signals
+ │ │ │ │ │ │ │ │ │ │ │ │ 4: (▼) queueRequest(), computeParams(), processStats()
+ ▼ ▼ ▼ ▼ │ ▼ │ ▼ │ ▼ │ ▼ (▲) setSensorControls, paramsComputed, metadataReady Signals
┌──────────────────┴────┴────┴────┴─────────┐ 5: stop(), unmapBuffers()
│ IPU3 IPA │
│ ┌───────────────────────┐ │
@@ -104,8 +104,8 @@ to operate when running:
- configure()
- queueRequest()
-- fillParamsBuffer()
-- processStatsBuffer()
+- computeParams()
+- processStats()
The configuration phase allows the pipeline-handler to inform the IPA of
the current stream configurations, which is then passed into each
@@ -119,7 +119,7 @@ When configured, the IPA is notified by the pipeline handler of the
Camera ``start()`` event, after which incoming requests will be queued
for processing, requiring a parameter buffer (``ipu3_uapi_params``) to
be populated for the ImgU. This is given to the IPA through
-``fillParamsBuffer()``, and then passed directly to each algorithm
+``computeParams()``, and then passed directly to each algorithm
through the ``prepare()`` call allowing the ISP configuration to be
updated for the needs of each component that the algorithm is
responsible for.
@@ -129,7 +129,7 @@ structure that it modifies, and it should take care to ensure that any
structure set by a use flag is fully initialised to suitable values.
The parameter buffer is returned to the pipeline handler through the
-``paramsBufferReady`` signal, and from there queued to the ImgU along
+``paramsComputed`` signal, and from there queued to the ImgU along
with a raw frame captured with the CIO2.
Post-frame completion
@@ -138,7 +138,7 @@ Post-frame completion
When the capture of an image is completed, and successfully processed
through the ImgU, the generated statistics buffer
(``ipu3_uapi_stats_3a``) is given to the IPA through
-``processStatsBuffer()``. This provides the IPA with an opportunity to
+``processStats()``. This provides the IPA with an opportunity to
examine the results of the ISP and run the calculations required by each
algorithm on the new data. The algorithms may require context from the
operations of other algorithms, for example, the AWB might choose to use
diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp
index 08ee6eb3..1cae08bf 100644
--- a/src/ipa/ipu3/ipu3.cpp
+++ b/src/ipa/ipu3/ipu3.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2020, Google Inc.
*
- * ipu3.cpp - IPU3 Image Processing Algorithms
+ * IPU3 Image Processing Algorithms
*/
#include <algorithm>
@@ -23,24 +23,22 @@
#include <libcamera/base/utils.h>
#include <libcamera/control_ids.h>
+#include <libcamera/controls.h>
#include <libcamera/framebuffer.h>
+#include <libcamera/geometry.h>
+#include <libcamera/request.h>
+
#include <libcamera/ipa/ipa_interface.h>
#include <libcamera/ipa/ipa_module_info.h>
#include <libcamera/ipa/ipu3_ipa_interface.h>
-#include <libcamera/request.h>
#include "libcamera/internal/mapped_framebuffer.h"
#include "libcamera/internal/yaml_parser.h"
-#include "algorithms/af.h"
-#include "algorithms/agc.h"
-#include "algorithms/algorithm.h"
-#include "algorithms/awb.h"
-#include "algorithms/blc.h"
-#include "algorithms/tone_mapping.h"
#include "libipa/camera_sensor_helper.h"
#include "ipa_context.h"
+#include "module.h"
/* Minimum grid width, expressed as a number of cells */
static constexpr uint32_t kMinGridWidth = 16;
@@ -89,14 +87,14 @@ namespace ipa::ipu3 {
* parameter buffer, and adapting the settings of the sensor attached to the
* IPU3 CIO2 through sensor-specific V4L2 controls.
*
- * In fillParamsBuffer(), we populate the ImgU parameter buffer with
+ * In computeParams(), we populate the ImgU parameter buffer with
* settings to configure the device in preparation for handling the frame
* queued in the Request.
*
* When the frame has completed processing, the ImgU will generate a statistics
- * buffer which is given to the IPA with processStatsBuffer(). In this we run the
+ * buffer which is given to the IPA with processStats(). In this we run the
* algorithms to parse the statistics and cache any results for the next
- * fillParamsBuffer() call.
+ * computeParams() call.
*
* The individual algorithms are split into modular components that are called
* iteratively to allow them to process statistics from the ImgU in the order
@@ -114,7 +112,7 @@ namespace ipa::ipu3 {
* blue gains to apply to generate a neutral grey frame overall.
*
* AGC is handled by calculating a histogram of the green channel to estimate an
- * analogue gain and shutter time which will provide a well exposed frame. A
+ * analogue gain and exposure time which will provide a well exposed frame. A
* low-pass IIR filter is used to smooth the changes to the sensor to reduce
* perceivable steps.
*
@@ -157,10 +155,10 @@ public:
void unmapBuffers(const std::vector<unsigned int> &ids) override;
void queueRequest(const uint32_t frame, const ControlList &controls) override;
- void fillParamsBuffer(const uint32_t frame, const uint32_t bufferId) override;
- void processStatsBuffer(const uint32_t frame, const int64_t frameTimestamp,
- const uint32_t bufferId,
- const ControlList &sensorControls) override;
+ void computeParams(const uint32_t frame, const uint32_t bufferId) override;
+ void processStats(const uint32_t frame, const int64_t frameTimestamp,
+ const uint32_t bufferId,
+ const ControlList &sensorControls) override;
protected:
std::string logPrefix() const override;
@@ -189,7 +187,7 @@ private:
};
IPAIPU3::IPAIPU3()
- : context_({ {}, {}, { kMaxFrameContexts } })
+ : context_(kMaxFrameContexts)
{
}
@@ -217,13 +215,13 @@ void IPAIPU3::updateSessionConfiguration(const ControlInfoMap &sensorControls)
/*
* When the AGC computes the new exposure values for a frame, it needs
- * to know the limits for shutter speed and analogue gain.
+ * to know the limits for exposure time and analogue gain.
* As it depends on the sensor, update it with the controls.
*
- * \todo take VBLANK into account for maximum shutter speed
+ * \todo take VBLANK into account for maximum exposure time
*/
- context_.configuration.agc.minShutterSpeed = minExposure * context_.configuration.sensor.lineDuration;
- context_.configuration.agc.maxShutterSpeed = maxExposure * context_.configuration.sensor.lineDuration;
+ context_.configuration.agc.minExposureTime = minExposure * context_.configuration.sensor.lineDuration;
+ context_.configuration.agc.maxExposureTime = maxExposure * context_.configuration.sensor.lineDuration;
context_.configuration.agc.minAnalogueGain = camHelper_->gain(minGain);
context_.configuration.agc.maxAnalogueGain = camHelper_->gain(maxGain);
}
@@ -287,6 +285,7 @@ void IPAIPU3::updateControls(const IPACameraSensorInfo &sensorInfo,
frameDurations[1],
frameDurations[2]);
+ controls.merge(context_.ctrlMap);
*ipaControls = ControlInfoMap(std::move(controls), controls::controls);
}
@@ -312,8 +311,8 @@ int IPAIPU3::init(const IPASettings &settings,
/* Clean context */
context_.configuration = {};
- context_.configuration.sensor.lineDuration = sensorInfo.minLineLength
- * 1.0s / sensorInfo.pixelRate;
+ context_.configuration.sensor.lineDuration =
+ sensorInfo.minLineLength * 1.0s / sensorInfo.pixelRate;
/* Load the tuning data file. */
File file(settings.configurationFile);
@@ -476,8 +475,8 @@ int IPAIPU3::configure(const IPAConfigInfo &configInfo,
context_.frameContexts.clear();
/* Initialise the sensor configuration. */
- context_.configuration.sensor.lineDuration = sensorInfo_.minLineLength
- * 1.0s / sensorInfo_.pixelRate;
+ context_.configuration.sensor.lineDuration =
+ sensorInfo_.minLineLength * 1.0s / sensorInfo_.pixelRate;
context_.configuration.sensor.size = sensorInfo_.outputSize;
/*
@@ -539,7 +538,7 @@ void IPAIPU3::unmapBuffers(const std::vector<unsigned int> &ids)
* Algorithms are expected to fill the IPU3 parameter buffer for the next
* frame given their most recent processing of the ImgU statistics.
*/
-void IPAIPU3::fillParamsBuffer(const uint32_t frame, const uint32_t bufferId)
+void IPAIPU3::computeParams(const uint32_t frame, const uint32_t bufferId)
{
auto it = buffers_.find(bufferId);
if (it == buffers_.end()) {
@@ -567,7 +566,7 @@ void IPAIPU3::fillParamsBuffer(const uint32_t frame, const uint32_t bufferId)
for (auto const &algo : algorithms())
algo->prepare(context_, frame, frameContext, params);
- paramsBufferReady.emit(frame);
+ paramsComputed.emit(frame);
}
/**
@@ -581,9 +580,9 @@ void IPAIPU3::fillParamsBuffer(const uint32_t frame, const uint32_t bufferId)
* statistics are passed to each algorithm module to run their calculations and
* update their state accordingly.
*/
-void IPAIPU3::processStatsBuffer(const uint32_t frame,
- [[maybe_unused]] const int64_t frameTimestamp,
- const uint32_t bufferId, const ControlList &sensorControls)
+void IPAIPU3::processStats(const uint32_t frame,
+ [[maybe_unused]] const int64_t frameTimestamp,
+ const uint32_t bufferId, const ControlList &sensorControls)
{
auto it = buffers_.find(bufferId);
if (it == buffers_.end()) {
@@ -672,7 +671,7 @@ extern "C" {
const struct IPAModuleInfo ipaModuleInfo = {
IPA_MODULE_API_VERSION,
1,
- "PipelineHandlerIPU3",
+ "ipu3",
"ipu3",
};
diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build
index 66c39843..34de6213 100644
--- a/src/ipa/ipu3/meson.build
+++ b/src/ipa/ipu3/meson.build
@@ -12,12 +12,10 @@ ipu3_ipa_sources = files([
ipu3_ipa_sources += ipu3_ipa_algorithms
-mod = shared_module(ipa_name,
- [ipu3_ipa_sources, libcamera_generated_ipa_headers],
+mod = shared_module(ipa_name, ipu3_ipa_sources,
name_prefix : '',
- include_directories : [ipa_includes, libipa_includes],
- dependencies : libcamera_private,
- link_with : libipa,
+ include_directories : [ipa_includes],
+ dependencies : [libcamera_private, libipa_dep],
install : true,
install_dir : ipa_install_dir)
diff --git a/src/ipa/ipu3/module.h b/src/ipa/ipu3/module.h
index d94fc459..60f65cc4 100644
--- a/src/ipa/ipu3/module.h
+++ b/src/ipa/ipu3/module.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2022, Ideas On Board
*
- * module.h - IPU3 IPA Module
+ * IPU3 IPA Module
*/
#pragma once
generated by cgit v1.2.1 (git 2.18.0) at 2025-06-01 12:36:34 +0000