summaryrefslogtreecommitdiff
path: root/src/ipa/ipu3
diff options
context:
space:
mode:
Diffstat (limited to 'src/ipa/ipu3')
-rw-r--r--src/ipa/ipu3/algorithms/af.cpp80
-rw-r--r--src/ipa/ipu3/algorithms/af.h14
-rw-r--r--src/ipa/ipu3/algorithms/agc.cpp321
-rw-r--r--src/ipa/ipu3/algorithms/agc.h35
-rw-r--r--src/ipa/ipu3/algorithms/algorithm.h2
-rw-r--r--src/ipa/ipu3/algorithms/awb.cpp179
-rw-r--r--src/ipa/ipu3/algorithms/awb.h12
-rw-r--r--src/ipa/ipu3/algorithms/blc.cpp14
-rw-r--r--src/ipa/ipu3/algorithms/blc.h6
-rw-r--r--src/ipa/ipu3/algorithms/tone_mapping.cpp26
-rw-r--r--src/ipa/ipu3/algorithms/tone_mapping.h11
-rw-r--r--src/ipa/ipu3/data/meson.build9
-rw-r--r--src/ipa/ipu3/data/uncalibrated.yaml11
-rw-r--r--src/ipa/ipu3/ipa_context.cpp54
-rw-r--r--src/ipa/ipu3/ipa_context.h24
-rw-r--r--src/ipa/ipu3/ipu3-ipa-design-guide.rst2
-rw-r--r--src/ipa/ipu3/ipu3.cpp170
-rw-r--r--src/ipa/ipu3/meson.build3
-rw-r--r--src/ipa/ipu3/module.h2
19 files changed, 462 insertions, 513 deletions
diff --git a/src/ipa/ipu3/algorithms/af.cpp b/src/ipa/ipu3/algorithms/af.cpp
index d07521a0..29eb7355 100644
--- a/src/ipa/ipu3/algorithms/af.cpp
+++ b/src/ipa/ipu3/algorithms/af.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Red Hat
*
- * af.cpp - IPU3 auto focus algorithm
+ * IPU3 auto focus algorithm
*/
#include "af.h"
@@ -114,19 +114,6 @@ Af::Af()
}
/**
- * \copydoc libcamera::ipa::Algorithm::prepare
- */
-void Af::prepare(IPAContext &context, ipu3_uapi_params *params)
-{
- const struct ipu3_uapi_grid_config &grid = context.configuration.af.afGrid;
- params->acc_param.af.grid_cfg = grid;
- params->acc_param.af.filter_config = afFilterConfigDefault;
-
- /* Enable AF processing block */
- params->use.acc_af = 1;
-}
-
-/**
* \brief Configure the Af given a configInfo
* \param[in] context The shared IPA context
* \param[in] configInfo The IPA configuration data
@@ -195,11 +182,27 @@ int Af::configure(IPAContext &context, const IPAConfigInfo &configInfo)
}
/**
+ * \copydoc libcamera::ipa::Algorithm::prepare
+ */
+void Af::prepare(IPAContext &context,
+ [[maybe_unused]] const uint32_t frame,
+ [[maybe_unused]] IPAFrameContext &frameContext,
+ ipu3_uapi_params *params)
+{
+ const struct ipu3_uapi_grid_config &grid = context.configuration.af.afGrid;
+ params->acc_param.af.grid_cfg = grid;
+ params->acc_param.af.filter_config = afFilterConfigDefault;
+
+ /* Enable AF processing block */
+ params->use.acc_af = 1;
+}
+
+/**
* \brief AF coarse scan
- *
- * Find a near focused image using a coarse step. The step is determined by coarseSearchStep.
- *
* \param[in] context The shared IPA context
+ *
+ * Find a near focused image using a coarse step. The step is determined by
+ * kCoarseSearchStep.
*/
void Af::afCoarseScan(IPAContext &context)
{
@@ -223,10 +226,9 @@ void Af::afCoarseScan(IPAContext &context)
/**
* \brief AF fine scan
+ * \param[in] context The shared IPA context
*
* Find an optimum lens position with moving 1 step for each search.
- *
- * \param[in] context The shared IPA context
*/
void Af::afFineScan(IPAContext &context)
{
@@ -244,10 +246,9 @@ void Af::afFineScan(IPAContext &context)
/**
* \brief AF reset
+ * \param[in] context The shared IPA context
*
* Reset all the parameters to start over the AF process.
- *
- * \param[in] context The shared IPA context
*/
void Af::afReset(IPAContext &context)
{
@@ -266,9 +267,9 @@ void Af::afReset(IPAContext &context)
}
/**
- * \brief AF variance comparison.
+ * \brief AF variance comparison
* \param[in] context The IPA context
- * \param min_step The VCM movement step.
+ * \param[in] min_step The VCM movement step
*
* We always pick the largest variance to replace the previous one. The image
* with a larger variance also indicates it is a clearer image than previous
@@ -321,7 +322,7 @@ bool Af::afScan(IPAContext &context, int min_step)
}
/**
- * \brief Determine the frame to be ignored.
+ * \brief Determine the frame to be ignored
* \return Return True if the frame should be ignored, false otherwise
*/
bool Af::afNeedIgnoreFrame()
@@ -334,7 +335,7 @@ bool Af::afNeedIgnoreFrame()
}
/**
- * \brief Reset frame ignore counter.
+ * \brief Reset frame ignore counter
*/
void Af::afIgnoreFrameReset()
{
@@ -343,9 +344,8 @@ void Af::afIgnoreFrameReset()
/**
* \brief Estimate variance
- * \param y_item The AF filter data set from the IPU3 statistics buffer
- * \param len The quantity of table item entries which are valid to process
- * \param isY1 Selects between filter Y1 or Y2 to calculate the variance
+ * \param[in] y_items The AF filter data set from the IPU3 statistics buffer
+ * \param[in] isY1 Selects between filter Y1 or Y2 to calculate the variance
*
* Calculate the mean of the data set provided by \a y_item, and then calculate
* the variance of that data set from the mean.
@@ -377,16 +377,16 @@ double Af::afEstimateVariance(Span<const y_table_item_t> y_items, bool isY1)
}
/**
- * \brief Determine out-of-focus situation.
- * \param context The IPA context.
+ * \brief Determine out-of-focus situation
+ * \param[in] context The IPA context
*
* Out-of-focus means that the variance change rate for a focused and a new
* variance is greater than a threshold.
*
* \return True if the variance threshold is crossed indicating lost focus,
- * false otherwise.
+ * false otherwise
*/
-bool Af::afIsOutOfFocus(IPAContext context)
+bool Af::afIsOutOfFocus(IPAContext &context)
{
const uint32_t diff_var = std::abs(currentVariance_ -
context.activeState.af.maxVariance);
@@ -404,10 +404,12 @@ bool Af::afIsOutOfFocus(IPAContext context)
}
/**
- * \brief Determine the max contrast image and lens position.
- * \param[in] context The IPA context.
+ * \brief Determine the max contrast image and lens position
+ * \param[in] context The IPA context
+ * \param[in] frame The frame context sequence number
* \param[in] frameContext The current frame context
- * \param[in] stats The statistics buffer of IPU3.
+ * \param[in] stats The statistics buffer of IPU3
+ * \param[out] metadata Metadata for the frame, to be filled by the algorithm
*
* Ideally, a clear image also has a relatively higher contrast. So, every
* image for each focus step should be tested to find an optimal focus step.
@@ -420,8 +422,10 @@ bool Af::afIsOutOfFocus(IPAContext context)
*
* [1] Hill Climbing Algorithm, https://en.wikipedia.org/wiki/Hill_climbing
*/
-void Af::process(IPAContext &context, [[maybe_unused]] IPAFrameContext *frameContext,
- const ipu3_uapi_stats_3a *stats)
+void Af::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
+ [[maybe_unused]] IPAFrameContext &frameContext,
+ const ipu3_uapi_stats_3a *stats,
+ [[maybe_unused]] ControlList &metadata)
{
/* Evaluate the AF buffer length */
uint32_t afRawBufferLen = context.configuration.af.afGrid.width *
@@ -450,6 +454,8 @@ void Af::process(IPAContext &context, [[maybe_unused]] IPAFrameContext *frameCon
}
}
+REGISTER_IPA_ALGORITHM(Af, "Af")
+
} /* namespace ipa::ipu3::algorithms */
} /* namespace libcamera */
diff --git a/src/ipa/ipu3/algorithms/af.h b/src/ipa/ipu3/algorithms/af.h
index ccf015f3..68126d46 100644
--- a/src/ipa/ipu3/algorithms/af.h
+++ b/src/ipa/ipu3/algorithms/af.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Red Hat
*
- * af.h - IPU3 Af algorithm
+ * IPU3 Af algorithm
*/
#pragma once
@@ -30,10 +30,14 @@ public:
Af();
~Af() = default;
- void prepare(IPAContext &context, ipu3_uapi_params *params) override;
int configure(IPAContext &context, const IPAConfigInfo &configInfo) override;
- void process(IPAContext &context, IPAFrameContext *frameContext,
- const ipu3_uapi_stats_3a *stats) override;
+ void prepare(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext,
+ ipu3_uapi_params *params) override;
+ void process(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext,
+ const ipu3_uapi_stats_3a *stats,
+ ControlList &metadata) override;
private:
void afCoarseScan(IPAContext &context);
@@ -44,7 +48,7 @@ private:
void afIgnoreFrameReset();
double afEstimateVariance(Span<const y_table_item_t> y_items, bool isY1);
- bool afIsOutOfFocus(IPAContext context);
+ bool afIsOutOfFocus(IPAContext &context);
/* VCM step configuration. It is the current setting of the VCM step. */
uint32_t focus_;
diff --git a/src/ipa/ipu3/algorithms/agc.cpp b/src/ipa/ipu3/algorithms/agc.cpp
index f16be534..0e0114f6 100644
--- a/src/ipa/ipu3/algorithms/agc.cpp
+++ b/src/ipa/ipu3/algorithms/agc.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * ipu3_agc.cpp - AGC/AEC mean-based control algorithm
+ * AGC/AEC mean-based control algorithm
*/
#include "agc.h"
@@ -14,6 +14,7 @@
#include <libcamera/base/log.h>
#include <libcamera/base/utils.h>
+#include <libcamera/control_ids.h>
#include <libcamera/ipa/core_ipa_interface.h>
#include "libipa/histogram.h"
@@ -46,9 +47,8 @@ namespace ipa::ipu3::algorithms {
LOG_DEFINE_CATEGORY(IPU3Agc)
-/* Limits for analogue gain values */
+/* Minimum limit for analogue gain value */
static constexpr double kMinAnalogueGain = 1.0;
-static constexpr double kMaxAnalogueGain = 8.0;
/* \todo Honour the FrameDurationLimits control instead of hardcoding a limit */
static constexpr utils::Duration kMaxShutterSpeed = 60ms;
@@ -56,24 +56,32 @@ static constexpr utils::Duration kMaxShutterSpeed = 60ms;
/* Histogram constants */
static constexpr uint32_t knumHistogramBins = 256;
-/* Target value to reach for the top 2% of the histogram */
-static constexpr double kEvGainTarget = 0.5;
-
-/* Number of frames to wait before calculating stats on minimum exposure */
-static constexpr uint32_t kNumStartupFrames = 10;
+Agc::Agc()
+ : minShutterSpeed_(0s), maxShutterSpeed_(0s)
+{
+}
-/*
- * Relative luminance target.
+/**
+ * \brief Initialise the AGC algorithm from tuning files
+ * \param[in] context The shared IPA context
+ * \param[in] tuningData The YamlObject containing Agc tuning data
+ *
+ * This function calls the base class' tuningData parsers to discover which
+ * control values are supported.
*
- * It's a number that's chosen so that, when the camera points at a grey
- * target, the resulting image brightness is considered right.
+ * \return 0 on success or errors from the base class
*/
-static constexpr double kRelativeLuminanceTarget = 0.16;
-
-Agc::Agc()
- : frameCount_(0), minShutterSpeed_(0s),
- maxShutterSpeed_(0s), filteredExposure_(0s)
+int Agc::init(IPAContext &context, const YamlObject &tuningData)
{
+ int ret;
+
+ ret = parseTuningData(tuningData);
+ if (ret)
+ return ret;
+
+ context.ctrlMap.merge(controls());
+
+ return 0;
}
/**
@@ -90,180 +98,66 @@ int Agc::configure(IPAContext &context,
IPAActiveState &activeState = context.activeState;
stride_ = configuration.grid.stride;
+ bdsGrid_ = configuration.grid.bdsGrid;
minShutterSpeed_ = configuration.agc.minShutterSpeed;
maxShutterSpeed_ = std::min(configuration.agc.maxShutterSpeed,
kMaxShutterSpeed);
minAnalogueGain_ = std::max(configuration.agc.minAnalogueGain, kMinAnalogueGain);
- maxAnalogueGain_ = std::min(configuration.agc.maxAnalogueGain, kMaxAnalogueGain);
+ maxAnalogueGain_ = configuration.agc.maxAnalogueGain;
/* Configure the default exposure and gain. */
- activeState.agc.gain = std::max(minAnalogueGain_, kMinAnalogueGain);
+ activeState.agc.gain = minAnalogueGain_;
activeState.agc.exposure = 10ms / configuration.sensor.lineDuration;
- frameCount_ = 0;
+ context.activeState.agc.constraintMode = constraintModes().begin()->first;
+ context.activeState.agc.exposureMode = exposureModeHelpers().begin()->first;
+
+ /* \todo Run this again when FrameDurationLimits is passed in */
+ setLimits(minShutterSpeed_, maxShutterSpeed_, minAnalogueGain_,
+ maxAnalogueGain_);
+ resetFrameCount();
+
return 0;
}
-/**
- * \brief Estimate the mean value of the top 2% of the histogram
- * \param[in] stats The statistics computed by the ImgU
- * \param[in] grid The grid used to store the statistics in the IPU3
- * \return The mean value of the top 2% of the histogram
- */
-double Agc::measureBrightness(const ipu3_uapi_stats_3a *stats,
- const ipu3_uapi_grid_config &grid) const
+Histogram Agc::parseStatistics(const ipu3_uapi_stats_3a *stats,
+ const ipu3_uapi_grid_config &grid)
{
- /* Initialise the histogram array */
uint32_t hist[knumHistogramBins] = { 0 };
+ rgbTriples_.clear();
+
for (unsigned int cellY = 0; cellY < grid.height; cellY++) {
for (unsigned int cellX = 0; cellX < grid.width; cellX++) {
uint32_t cellPosition = cellY * stride_ + cellX;
const ipu3_uapi_awb_set_item *cell =
reinterpret_cast<const ipu3_uapi_awb_set_item *>(
- &stats->awb_raw_buffer.meta_data[cellPosition]
- );
+ &stats->awb_raw_buffer.meta_data[cellPosition]);
+
+ rgbTriples_.push_back({
+ cell->R_avg,
+ (cell->Gr_avg + cell->Gb_avg) / 2,
+ cell->B_avg
+ });
- uint8_t gr = cell->Gr_avg;
- uint8_t gb = cell->Gb_avg;
/*
* Store the average green value to estimate the
* brightness. Even the overexposed pixels are
* taken into account.
*/
- hist[(gr + gb) / 2]++;
+ hist[(cell->Gr_avg + cell->Gb_avg) / 2]++;
}
}
- /* Estimate the quantile mean of the top 2% of the histogram. */
- return Histogram(Span<uint32_t>(hist)).interQuantileMean(0.98, 1.0);
-}
-
-/**
- * \brief Apply a filter on the exposure value to limit the speed of changes
- * \param[in] exposureValue The target exposure from the AGC algorithm
- *
- * The speed of the filter is adaptive, and will produce the target quicker
- * during startup, or when the target exposure is within 20% of the most recent
- * filter output.
- *
- * \return The filtered exposure
- */
-utils::Duration Agc::filterExposure(utils::Duration exposureValue)
-{
- double speed = 0.2;
-
- /* Adapt instantly if we are in startup phase. */
- if (frameCount_ < kNumStartupFrames)
- speed = 1.0;
-
- /*
- * If we are close to the desired result, go faster to avoid making
- * multiple micro-adjustments.
- * \todo Make this customisable?
- */
- if (filteredExposure_ < 1.2 * exposureValue &&
- filteredExposure_ > 0.8 * exposureValue)
- speed = sqrt(speed);
-
- filteredExposure_ = speed * exposureValue +
- filteredExposure_ * (1.0 - speed);
-
- LOG(IPU3Agc, Debug) << "After filtering, exposure " << filteredExposure_;
-
- return filteredExposure_;
-}
-
-/**
- * \brief Estimate the new exposure and gain values
- * \param[inout] frameContext The shared IPA frame Context
- * \param[in] yGain The gain calculated based on the relative luminance target
- * \param[in] iqMeanGain The gain calculated based on the relative luminance target
- */
-void Agc::computeExposure(IPAContext &context, IPAFrameContext *frameContext,
- double yGain, double iqMeanGain)
-{
- const IPASessionConfiguration &configuration = context.configuration;
- /* Get the effective exposure and gain applied on the sensor. */
- uint32_t exposure = frameContext->sensor.exposure;
- double analogueGain = frameContext->sensor.gain;
-
- /* Use the highest of the two gain estimates. */
- double evGain = std::max(yGain, iqMeanGain);
-
- /* Consider within 1% of the target as correctly exposed */
- if (utils::abs_diff(evGain, 1.0) < 0.01)
- LOG(IPU3Agc, Debug) << "We are well exposed (evGain = "
- << evGain << ")";
-
- /* extracted from Rpi::Agc::computeTargetExposure */
-
- /* Calculate the shutter time in seconds */
- utils::Duration currentShutter = exposure * configuration.sensor.lineDuration;
-
- /*
- * Update the exposure value for the next computation using the values
- * of exposure and gain really used by the sensor.
- */
- utils::Duration effectiveExposureValue = currentShutter * analogueGain;
-
- LOG(IPU3Agc, Debug) << "Actual total exposure " << currentShutter * analogueGain
- << " Shutter speed " << currentShutter
- << " Gain " << analogueGain
- << " Needed ev gain " << evGain;
-
- /*
- * Calculate the current exposure value for the scene as the latest
- * exposure value applied multiplied by the new estimated gain.
- */
- utils::Duration exposureValue = effectiveExposureValue * evGain;
-
- /* Clamp the exposure value to the min and max authorized */
- utils::Duration maxTotalExposure = maxShutterSpeed_ * maxAnalogueGain_;
- exposureValue = std::min(exposureValue, maxTotalExposure);
- LOG(IPU3Agc, Debug) << "Target total exposure " << exposureValue
- << ", maximum is " << maxTotalExposure;
-
- /*
- * Filter the exposure.
- * \todo: estimate if we need to desaturate
- */
- exposureValue = filterExposure(exposureValue);
-
- /*
- * Divide the exposure value as new exposure and gain values.
- *
- * Push the shutter time up to the maximum first, and only then
- * increase the gain.
- */
- utils::Duration shutterTime =
- std::clamp<utils::Duration>(exposureValue / minAnalogueGain_,
- minShutterSpeed_, maxShutterSpeed_);
- double stepGain = std::clamp(exposureValue / shutterTime,
- minAnalogueGain_, maxAnalogueGain_);
- LOG(IPU3Agc, Debug) << "Divided up shutter and gain are "
- << shutterTime << " and "
- << stepGain;
-
- IPAActiveState &activeState = context.activeState;
- /* Update the estimated exposure and gain. */
- activeState.agc.exposure = shutterTime / configuration.sensor.lineDuration;
- activeState.agc.gain = stepGain;
+ return Histogram(Span<uint32_t>(hist));
}
/**
* \brief Estimate the relative luminance of the frame with a given gain
- * \param[in] frameContext The shared IPA frame context
- * \param[in] grid The grid used to store the statistics in the IPU3
- * \param[in] stats The IPU3 statistics and ISP results
- * \param[in] gain The gain to apply to the frame
- * \return The relative luminance
- *
- * This function estimates the average relative luminance of the frame that
- * would be output by the sensor if an additional \a gain was applied.
+ * \param[in] gain The gain to apply in estimating luminance
*
* The estimation is based on the AWB statistics for the current frame. Red,
* green and blue averages for all cells are first multiplied by the gain, and
@@ -278,91 +172,86 @@ void Agc::computeExposure(IPAContext &context, IPAFrameContext *frameContext,
*
* More detailed information can be found in:
* https://en.wikipedia.org/wiki/Relative_luminance
+ *
+ * \return The relative luminance of the frame
*/
-double Agc::estimateLuminance(IPAActiveState &activeState,
- const ipu3_uapi_grid_config &grid,
- const ipu3_uapi_stats_3a *stats,
- double gain)
+double Agc::estimateLuminance(double gain) const
{
double redSum = 0, greenSum = 0, blueSum = 0;
- /* Sum the per-channel averages, saturated to 255. */
- for (unsigned int cellY = 0; cellY < grid.height; cellY++) {
- for (unsigned int cellX = 0; cellX < grid.width; cellX++) {
- uint32_t cellPosition = cellY * stride_ + cellX;
-
- const ipu3_uapi_awb_set_item *cell =
- reinterpret_cast<const ipu3_uapi_awb_set_item *>(
- &stats->awb_raw_buffer.meta_data[cellPosition]
- );
- const uint8_t G_avg = (cell->Gr_avg + cell->Gb_avg) / 2;
-
- redSum += std::min(cell->R_avg * gain, 255.0);
- greenSum += std::min(G_avg * gain, 255.0);
- blueSum += std::min(cell->B_avg * gain, 255.0);
- }
+ for (unsigned int i = 0; i < rgbTriples_.size(); i++) {
+ redSum += std::min(std::get<0>(rgbTriples_[i]) * gain, 255.0);
+ greenSum += std::min(std::get<1>(rgbTriples_[i]) * gain, 255.0);
+ blueSum += std::min(std::get<2>(rgbTriples_[i]) * gain, 255.0);
}
- /*
- * Apply the AWB gains to approximate colours correctly, use the Rec.
- * 601 formula to calculate the relative luminance, and normalize it.
- */
- double ySum = redSum * activeState.awb.gains.red * 0.299
- + greenSum * activeState.awb.gains.green * 0.587
- + blueSum * activeState.awb.gains.blue * 0.114;
+ double ySum = redSum * rGain_ * 0.299
+ + greenSum * gGain_ * 0.587
+ + blueSum * bGain_ * 0.114;
- return ySum / (grid.height * grid.width) / 255;
+ return ySum / (bdsGrid_.height * bdsGrid_.width) / 255;
}
/**
* \brief Process IPU3 statistics, and run AGC operations
* \param[in] context The shared IPA context
+ * \param[in] frame The current frame sequence number
* \param[in] frameContext The current frame context
* \param[in] stats The IPU3 statistics and ISP results
+ * \param[out] metadata Metadata for the frame, to be filled by the algorithm
*
* Identify the current image brightness, and use that to estimate the optimal
* new exposure and gain for the scene.
*/
-void Agc::process(IPAContext &context, [[maybe_unused]] IPAFrameContext *frameContext,
- const ipu3_uapi_stats_3a *stats)
+void Agc::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
+ IPAFrameContext &frameContext,
+ const ipu3_uapi_stats_3a *stats,
+ ControlList &metadata)
{
- /*
- * Estimate the gain needed to have the proportion of pixels in a given
- * desired range. iqMean is the mean value of the top 2% of the
- * cumulative histogram, and we want it to be as close as possible to a
- * configured target.
- */
- double iqMean = measureBrightness(stats, context.configuration.grid.bdsGrid);
- double iqMeanGain = kEvGainTarget * knumHistogramBins / iqMean;
+ Histogram hist = parseStatistics(stats, context.configuration.grid.bdsGrid);
+ rGain_ = context.activeState.awb.gains.red;
+ gGain_ = context.activeState.awb.gains.blue;
+ bGain_ = context.activeState.awb.gains.green;
/*
- * Estimate the gain needed to achieve a relative luminance target. To
- * account for non-linearity caused by saturation, the value needs to be
- * estimated in an iterative process, as multiplying by a gain will not
- * increase the relative luminance by the same factor if some image
- * regions are saturated.
+ * The Agc algorithm needs to know the effective exposure value that was
+ * applied to the sensor when the statistics were collected.
*/
- double yGain = 1.0;
- double yTarget = kRelativeLuminanceTarget;
-
- for (unsigned int i = 0; i < 8; i++) {
- double yValue = estimateLuminance(context.activeState,
- context.configuration.grid.bdsGrid,
- stats, yGain);
- double extraGain = std::min(10.0, yTarget / (yValue + .001));
-
- yGain *= extraGain;
- LOG(IPU3Agc, Debug) << "Y value: " << yValue
- << ", Y target: " << yTarget
- << ", gives gain " << yGain;
- if (extraGain < 1.01)
- break;
- }
+ utils::Duration exposureTime = context.configuration.sensor.lineDuration
+ * frameContext.sensor.exposure;
+ double analogueGain = frameContext.sensor.gain;
+ utils::Duration effectiveExposureValue = exposureTime * analogueGain;
+
+ utils::Duration shutterTime;
+ double aGain, dGain;
+ std::tie(shutterTime, aGain, dGain) =
+ calculateNewEv(context.activeState.agc.constraintMode,
+ context.activeState.agc.exposureMode, hist,
+ effectiveExposureValue);
+
+ LOG(IPU3Agc, Debug)
+ << "Divided up shutter, analogue gain and digital gain are "
+ << shutterTime << ", " << aGain << " and " << dGain;
+
+ IPAActiveState &activeState = context.activeState;
+ /* Update the estimated exposure and gain. */
+ activeState.agc.exposure = shutterTime / context.configuration.sensor.lineDuration;
+ activeState.agc.gain = aGain;
+
+ metadata.set(controls::AnalogueGain, frameContext.sensor.gain);
+ metadata.set(controls::ExposureTime, exposureTime.get<std::micro>());
+
+ /* \todo Use VBlank value calculated from each frame exposure. */
+ uint32_t vTotal = context.configuration.sensor.size.height
+ + context.configuration.sensor.defVBlank;
+ utils::Duration frameDuration = context.configuration.sensor.lineDuration
+ * vTotal;
+ metadata.set(controls::FrameDuration, frameDuration.get<std::micro>());
- computeExposure(context, frameContext, yGain, iqMeanGain);
- frameCount_++;
}
+REGISTER_IPA_ALGORITHM(Agc, "Agc")
+
} /* namespace ipa::ipu3::algorithms */
} /* namespace libcamera */
diff --git a/src/ipa/ipu3/algorithms/agc.h b/src/ipa/ipu3/algorithms/agc.h
index 105ae0f2..411f4da0 100644
--- a/src/ipa/ipu3/algorithms/agc.h
+++ b/src/ipa/ipu3/algorithms/agc.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * agc.h - IPU3 AGC/AEC mean-based control algorithm
+ * IPU3 AGC/AEC mean-based control algorithm
*/
#pragma once
@@ -13,6 +13,9 @@
#include <libcamera/geometry.h>
+#include "libipa/agc_mean_luminance.h"
+#include "libipa/histogram.h"
+
#include "algorithm.h"
namespace libcamera {
@@ -21,28 +24,23 @@ struct IPACameraSensorInfo;
namespace ipa::ipu3::algorithms {
-class Agc : public Algorithm
+class Agc : public Algorithm, public AgcMeanLuminance
{
public:
Agc();
~Agc() = default;
+ int init(IPAContext &context, const YamlObject &tuningData) override;
int configure(IPAContext &context, const IPAConfigInfo &configInfo) override;
- void process(IPAContext &context, IPAFrameContext *frameContext,
- const ipu3_uapi_stats_3a *stats) override;
+ void process(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext,
+ const ipu3_uapi_stats_3a *stats,
+ ControlList &metadata) override;
private:
- double measureBrightness(const ipu3_uapi_stats_3a *stats,
- const ipu3_uapi_grid_config &grid) const;
- utils::Duration filterExposure(utils::Duration currentExposure);
- void computeExposure(IPAContext &context, IPAFrameContext *frameContext,
- double yGain, double iqMeanGain);
- double estimateLuminance(IPAActiveState &activeState,
- const ipu3_uapi_grid_config &grid,
- const ipu3_uapi_stats_3a *stats,
- double gain);
-
- uint64_t frameCount_;
+ double estimateLuminance(double gain) const override;
+ Histogram parseStatistics(const ipu3_uapi_stats_3a *stats,
+ const ipu3_uapi_grid_config &grid);
utils::Duration minShutterSpeed_;
utils::Duration maxShutterSpeed_;
@@ -50,9 +48,12 @@ private:
double minAnalogueGain_;
double maxAnalogueGain_;
- utils::Duration filteredExposure_;
-
uint32_t stride_;
+ double rGain_;
+ double gGain_;
+ double bGain_;
+ ipu3_uapi_grid_config bdsGrid_;
+ std::vector<std::tuple<uint8_t, uint8_t, uint8_t>> rgbTriples_;
};
} /* namespace ipa::ipu3::algorithms */
diff --git a/src/ipa/ipu3/algorithms/algorithm.h b/src/ipa/ipu3/algorithms/algorithm.h
index ae134a94..c7801f93 100644
--- a/src/ipa/ipu3/algorithms/algorithm.h
+++ b/src/ipa/ipu3/algorithms/algorithm.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * algorithm.h - IPU3 control algorithm interface
+ * IPU3 control algorithm interface
*/
#pragma once
diff --git a/src/ipa/ipu3/algorithms/awb.cpp b/src/ipa/ipu3/algorithms/awb.cpp
index 70426722..4d6e3994 100644
--- a/src/ipa/ipu3/algorithms/awb.cpp
+++ b/src/ipa/ipu3/algorithms/awb.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * awb.cpp - AWB control algorithm
+ * AWB control algorithm
*/
#include "awb.h"
@@ -11,6 +11,8 @@
#include <libcamera/base/log.h>
+#include <libcamera/control_ids.h>
+
/**
* \file awb.h
*/
@@ -216,6 +218,89 @@ int Awb::configure(IPAContext &context,
return 0;
}
+constexpr uint16_t Awb::threshold(float value)
+{
+ /* AWB thresholds are in the range [0, 8191] */
+ return value * 8191;
+}
+
+constexpr uint16_t Awb::gainValue(double gain)
+{
+ /*
+ * The colour gains applied by the BNR for the four channels (Gr, R, B
+ * and Gb) are expressed in the parameters structure as 16-bit integers
+ * that store a fixed-point U3.13 value in the range [0, 8[.
+ *
+ * The real gain value is equal to the gain parameter plus one, i.e.
+ *
+ * Pout = Pin * (1 + gain / 8192)
+ *
+ * where 'Pin' is the input pixel value, 'Pout' the output pixel value,
+ * and 'gain' the gain in the parameters structure as a 16-bit integer.
+ */
+ return std::clamp((gain - 1.0) * 8192, 0.0, 65535.0);
+}
+
+/**
+ * \copydoc libcamera::ipa::Algorithm::prepare
+ */
+void Awb::prepare(IPAContext &context,
+ [[maybe_unused]] const uint32_t frame,
+ [[maybe_unused]] IPAFrameContext &frameContext,
+ ipu3_uapi_params *params)
+{
+ /*
+ * Green saturation thresholds are reduced because we are using the
+ * green channel only in the exposure computation.
+ */
+ params->acc_param.awb.config.rgbs_thr_r = threshold(1.0);
+ params->acc_param.awb.config.rgbs_thr_gr = threshold(0.9);
+ params->acc_param.awb.config.rgbs_thr_gb = threshold(0.9);
+ params->acc_param.awb.config.rgbs_thr_b = threshold(1.0);
+
+ /*
+ * Enable saturation inclusion on thr_b for ImgU to update the
+ * ipu3_uapi_awb_set_item->sat_ratio field.
+ */
+ params->acc_param.awb.config.rgbs_thr_b |= IPU3_UAPI_AWB_RGBS_THR_B_INCL_SAT |
+ IPU3_UAPI_AWB_RGBS_THR_B_EN;
+
+ const ipu3_uapi_grid_config &grid = context.configuration.grid.bdsGrid;
+
+ params->acc_param.awb.config.grid = context.configuration.grid.bdsGrid;
+
+ /*
+ * Optical center is column start (respectively row start) of the
+ * cell of interest minus its X center (respectively Y center).
+ *
+ * For the moment use BDS as a first approximation, but it should
+ * be calculated based on Shading (SHD) parameters.
+ */
+ params->acc_param.bnr = imguCssBnrDefaults;
+ Size &bdsOutputSize = context.configuration.grid.bdsOutputSize;
+ params->acc_param.bnr.column_size = bdsOutputSize.width;
+ params->acc_param.bnr.opt_center.x_reset = grid.x_start - (bdsOutputSize.width / 2);
+ params->acc_param.bnr.opt_center.y_reset = grid.y_start - (bdsOutputSize.height / 2);
+ params->acc_param.bnr.opt_center_sqr.x_sqr_reset = params->acc_param.bnr.opt_center.x_reset
+ * params->acc_param.bnr.opt_center.x_reset;
+ params->acc_param.bnr.opt_center_sqr.y_sqr_reset = params->acc_param.bnr.opt_center.y_reset
+ * params->acc_param.bnr.opt_center.y_reset;
+
+ params->acc_param.bnr.wb_gains.gr = gainValue(context.activeState.awb.gains.green);
+ params->acc_param.bnr.wb_gains.r = gainValue(context.activeState.awb.gains.red);
+ params->acc_param.bnr.wb_gains.b = gainValue(context.activeState.awb.gains.blue);
+ params->acc_param.bnr.wb_gains.gb = gainValue(context.activeState.awb.gains.green);
+
+ LOG(IPU3Awb, Debug) << "Color temperature estimated: " << asyncResults_.temperatureK;
+
+ /* The CCM matrix may change when color temperature will be used */
+ params->acc_param.ccm = imguCssCcmDefault;
+
+ params->use.acc_awb = 1;
+ params->use.acc_bnr = 1;
+ params->use.acc_ccm = 1;
+}
+
/**
* The function estimates the correlated color temperature using
* from RGB color space input.
@@ -387,8 +472,10 @@ void Awb::calculateWBGains(const ipu3_uapi_stats_3a *stats)
/**
* \copydoc libcamera::ipa::Algorithm::process
*/
-void Awb::process(IPAContext &context, [[maybe_unused]] IPAFrameContext *frameContext,
- const ipu3_uapi_stats_3a *stats)
+void Awb::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
+ [[maybe_unused]] IPAFrameContext &frameContext,
+ const ipu3_uapi_stats_3a *stats,
+ [[maybe_unused]] ControlList &metadata)
{
calculateWBGains(stats);
@@ -401,87 +488,17 @@ void Awb::process(IPAContext &context, [[maybe_unused]] IPAFrameContext *frameCo
context.activeState.awb.gains.green = asyncResults_.greenGain;
context.activeState.awb.gains.red = asyncResults_.redGain;
context.activeState.awb.temperatureK = asyncResults_.temperatureK;
-}
-constexpr uint16_t Awb::threshold(float value)
-{
- /* AWB thresholds are in the range [0, 8191] */
- return value * 8191;
+ metadata.set(controls::AwbEnable, true);
+ metadata.set(controls::ColourGains, {
+ static_cast<float>(context.activeState.awb.gains.red),
+ static_cast<float>(context.activeState.awb.gains.blue)
+ });
+ metadata.set(controls::ColourTemperature,
+ context.activeState.awb.temperatureK);
}
-constexpr uint16_t Awb::gainValue(double gain)
-{
- /*
- * The colour gains applied by the BNR for the four channels (Gr, R, B
- * and Gb) are expressed in the parameters structure as 16-bit integers
- * that store a fixed-point U3.13 value in the range [0, 8[.
- *
- * The real gain value is equal to the gain parameter plus one, i.e.
- *
- * Pout = Pin * (1 + gain / 8192)
- *
- * where 'Pin' is the input pixel value, 'Pout' the output pixel value,
- * and 'gain' the gain in the parameters structure as a 16-bit integer.
- */
- return std::clamp((gain - 1.0) * 8192, 0.0, 65535.0);
-}
-
-/**
- * \copydoc libcamera::ipa::Algorithm::prepare
- */
-void Awb::prepare(IPAContext &context, ipu3_uapi_params *params)
-{
- /*
- * Green saturation thresholds are reduced because we are using the
- * green channel only in the exposure computation.
- */
- params->acc_param.awb.config.rgbs_thr_r = threshold(1.0);
- params->acc_param.awb.config.rgbs_thr_gr = threshold(0.9);
- params->acc_param.awb.config.rgbs_thr_gb = threshold(0.9);
- params->acc_param.awb.config.rgbs_thr_b = threshold(1.0);
-
- /*
- * Enable saturation inclusion on thr_b for ImgU to update the
- * ipu3_uapi_awb_set_item->sat_ratio field.
- */
- params->acc_param.awb.config.rgbs_thr_b |= IPU3_UAPI_AWB_RGBS_THR_B_INCL_SAT |
- IPU3_UAPI_AWB_RGBS_THR_B_EN;
-
- const ipu3_uapi_grid_config &grid = context.configuration.grid.bdsGrid;
-
- params->acc_param.awb.config.grid = context.configuration.grid.bdsGrid;
-
- /*
- * Optical center is column start (respectively row start) of the
- * cell of interest minus its X center (respectively Y center).
- *
- * For the moment use BDS as a first approximation, but it should
- * be calculated based on Shading (SHD) parameters.
- */
- params->acc_param.bnr = imguCssBnrDefaults;
- Size &bdsOutputSize = context.configuration.grid.bdsOutputSize;
- params->acc_param.bnr.column_size = bdsOutputSize.width;
- params->acc_param.bnr.opt_center.x_reset = grid.x_start - (bdsOutputSize.width / 2);
- params->acc_param.bnr.opt_center.y_reset = grid.y_start - (bdsOutputSize.height / 2);
- params->acc_param.bnr.opt_center_sqr.x_sqr_reset = params->acc_param.bnr.opt_center.x_reset
- * params->acc_param.bnr.opt_center.x_reset;
- params->acc_param.bnr.opt_center_sqr.y_sqr_reset = params->acc_param.bnr.opt_center.y_reset
- * params->acc_param.bnr.opt_center.y_reset;
-
- params->acc_param.bnr.wb_gains.gr = gainValue(context.activeState.awb.gains.green);
- params->acc_param.bnr.wb_gains.r = gainValue(context.activeState.awb.gains.red);
- params->acc_param.bnr.wb_gains.b = gainValue(context.activeState.awb.gains.blue);
- params->acc_param.bnr.wb_gains.gb = gainValue(context.activeState.awb.gains.green);
-
- LOG(IPU3Awb, Debug) << "Color temperature estimated: " << asyncResults_.temperatureK;
-
- /* The CCM matrix may change when color temperature will be used */
- params->acc_param.ccm = imguCssCcmDefault;
-
- params->use.acc_awb = 1;
- params->use.acc_bnr = 1;
- params->use.acc_ccm = 1;
-}
+REGISTER_IPA_ALGORITHM(Awb, "Awb")
} /* namespace ipa::ipu3::algorithms */
diff --git a/src/ipa/ipu3/algorithms/awb.h b/src/ipa/ipu3/algorithms/awb.h
index 0acd2148..c0202823 100644
--- a/src/ipa/ipu3/algorithms/awb.h
+++ b/src/ipa/ipu3/algorithms/awb.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Ideas On Board
*
- * awb.h - IPU3 AWB control algorithm
+ * IPU3 AWB control algorithm
*/
#pragma once
@@ -39,9 +39,13 @@ public:
~Awb();
int configure(IPAContext &context, const IPAConfigInfo &configInfo) override;
- void prepare(IPAContext &context, ipu3_uapi_params *params) override;
- void process(IPAContext &context, IPAFrameContext *frameContext,
- const ipu3_uapi_stats_3a *stats) override;
+ void prepare(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext,
+ ipu3_uapi_params *params) override;
+ void process(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext,
+ const ipu3_uapi_stats_3a *stats,
+ ControlList &metadata) override;
private:
/* \todo Make these structs available to all the ISPs ? */
diff --git a/src/ipa/ipu3/algorithms/blc.cpp b/src/ipa/ipu3/algorithms/blc.cpp
index 78ab7bff..257f40e2 100644
--- a/src/ipa/ipu3/algorithms/blc.cpp
+++ b/src/ipa/ipu3/algorithms/blc.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google inc.
*
- * blc.cpp - IPU3 Black Level Correction control
+ * IPU3 Black Level Correction control
*/
#include "blc.h"
@@ -38,14 +38,18 @@ BlackLevelCorrection::BlackLevelCorrection()
/**
* \brief Fill in the parameter structure, and enable black level correction
- * \param context The shared IPA context
- * \param params The IPU3 parameters
+ * \param[in] context The shared IPA context
+ * \param[in] frame The frame context sequence number
+ * \param[in] frameContext The FrameContext for this frame
+ * \param[out] params The IPU3 parameters
*
* Populate the IPU3 parameter structure with the correction values for each
* channel and enable the corresponding ImgU block processing.
*/
void BlackLevelCorrection::prepare([[maybe_unused]] IPAContext &context,
- ipu3_uapi_params *params)
+ [[maybe_unused]] const uint32_t frame,
+ [[maybe_unused]] IPAFrameContext &frameContext,
+ ipu3_uapi_params *params)
{
/*
* The Optical Black Level correction values
@@ -62,6 +66,8 @@ void BlackLevelCorrection::prepare([[maybe_unused]] IPAContext &context,
params->use.obgrid_param = 1;
}
+REGISTER_IPA_ALGORITHM(BlackLevelCorrection, "BlackLevelCorrection")
+
} /* namespace ipa::ipu3::algorithms */
} /* namespace libcamera */
diff --git a/src/ipa/ipu3/algorithms/blc.h b/src/ipa/ipu3/algorithms/blc.h
index d8da1748..62748045 100644
--- a/src/ipa/ipu3/algorithms/blc.h
+++ b/src/ipa/ipu3/algorithms/blc.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google inc.
*
- * black_correction.h - IPU3 Black Level Correction control
+ * IPU3 Black Level Correction control
*/
#pragma once
@@ -18,7 +18,9 @@ class BlackLevelCorrection : public Algorithm
public:
BlackLevelCorrection();
- void prepare(IPAContext &context, ipu3_uapi_params *params) override;
+ void prepare(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext,
+ ipu3_uapi_params *params) override;
};
} /* namespace ipa::ipu3::algorithms */
diff --git a/src/ipa/ipu3/algorithms/tone_mapping.cpp b/src/ipa/ipu3/algorithms/tone_mapping.cpp
index f86e79b2..160338c1 100644
--- a/src/ipa/ipu3/algorithms/tone_mapping.cpp
+++ b/src/ipa/ipu3/algorithms/tone_mapping.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google inc.
*
- * tone_mapping.cpp - IPU3 ToneMapping and Gamma control
+ * IPU3 ToneMapping and Gamma control
*/
#include "tone_mapping.h"
@@ -49,13 +49,17 @@ int ToneMapping::configure(IPAContext &context,
/**
* \brief Fill in the parameter structure, and enable gamma control
- * \param context The shared IPA context
- * \param params The IPU3 parameters
+ * \param[in] context The shared IPA context
+ * \param[in] frame The frame context sequence number
+ * \param[in] frameContext The FrameContext for this frame
+ * \param[out] params The IPU3 parameters
*
* Populate the IPU3 parameter structure with our tone mapping look up table and
* enable the gamma control module in the processing blocks.
*/
void ToneMapping::prepare([[maybe_unused]] IPAContext &context,
+ [[maybe_unused]] const uint32_t frame,
+ [[maybe_unused]] IPAFrameContext &frameContext,
ipu3_uapi_params *params)
{
/* Copy the calculated LUT into the parameters buffer. */
@@ -71,15 +75,19 @@ void ToneMapping::prepare([[maybe_unused]] IPAContext &context,
/**
* \brief Calculate the tone mapping look up table
- * \param context The shared IPA context
- * \param frameContext The current frame context
- * \param stats The IPU3 statistics and ISP results
+ * \param[in] context The shared IPA context
+ * \param[in] frame The current frame sequence number
+ * \param[in] frameContext The current frame context
+ * \param[in] stats The IPU3 statistics and ISP results
+ * \param[out] metadata Metadata for the frame, to be filled by the algorithm
*
* The tone mapping look up table is generated as an inverse power curve from
* our gamma setting.
*/
-void ToneMapping::process(IPAContext &context, [[maybe_unused]] IPAFrameContext *frameContext,
- [[maybe_unused]] const ipu3_uapi_stats_3a *stats)
+void ToneMapping::process(IPAContext &context, [[maybe_unused]] const uint32_t frame,
+ [[maybe_unused]] IPAFrameContext &frameContext,
+ [[maybe_unused]] const ipu3_uapi_stats_3a *stats,
+ [[maybe_unused]] ControlList &metadata)
{
/*
* Hardcode gamma to 1.1 as a default for now.
@@ -105,6 +113,8 @@ void ToneMapping::process(IPAContext &context, [[maybe_unused]] IPAFrameContext
context.activeState.toneMapping.gamma = gamma_;
}
+REGISTER_IPA_ALGORITHM(ToneMapping, "ToneMapping")
+
} /* namespace ipa::ipu3::algorithms */
} /* namespace libcamera */
diff --git a/src/ipa/ipu3/algorithms/tone_mapping.h b/src/ipa/ipu3/algorithms/tone_mapping.h
index d7d48006..b2b38010 100644
--- a/src/ipa/ipu3/algorithms/tone_mapping.h
+++ b/src/ipa/ipu3/algorithms/tone_mapping.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google inc.
*
- * tone_mapping.h - IPU3 ToneMapping and Gamma control
+ * IPU3 ToneMapping and Gamma control
*/
#pragma once
@@ -19,9 +19,12 @@ public:
ToneMapping();
int configure(IPAContext &context, const IPAConfigInfo &configInfo) override;
- void prepare(IPAContext &context, ipu3_uapi_params *params) override;
- void process(IPAContext &context, IPAFrameContext *frameContext,
- const ipu3_uapi_stats_3a *stats) override;
+ void prepare(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext, ipu3_uapi_params *params) override;
+ void process(IPAContext &context, const uint32_t frame,
+ IPAFrameContext &frameContext,
+ const ipu3_uapi_stats_3a *stats,
+ ControlList &metadata) override;
private:
double gamma_;
diff --git a/src/ipa/ipu3/data/meson.build b/src/ipa/ipu3/data/meson.build
new file mode 100644
index 00000000..0f7cd5c6
--- /dev/null
+++ b/src/ipa/ipu3/data/meson.build
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: CC0-1.0
+
+conf_files = files([
+ 'uncalibrated.yaml',
+])
+
+install_data(conf_files,
+ install_dir : ipa_data_dir / 'ipu3',
+ install_tag : 'runtime')
diff --git a/src/ipa/ipu3/data/uncalibrated.yaml b/src/ipa/ipu3/data/uncalibrated.yaml
new file mode 100644
index 00000000..794ab3ed
--- /dev/null
+++ b/src/ipa/ipu3/data/uncalibrated.yaml
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: CC0-1.0
+%YAML 1.1
+---
+version: 1
+algorithms:
+ - Af:
+ - Agc:
+ - Awb:
+ - BlackLevelCorrection:
+ - ToneMapping:
+...
diff --git a/src/ipa/ipu3/ipa_context.cpp b/src/ipa/ipu3/ipa_context.cpp
index 13cdb835..917d0654 100644
--- a/src/ipa/ipu3/ipa_context.cpp
+++ b/src/ipa/ipu3/ipa_context.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2021, Google Inc.
*
- * ipa_context.cpp - IPU3 IPA Context
+ * IPU3 IPA Context
*/
#include "ipa_context.h"
@@ -36,22 +36,6 @@ namespace libcamera::ipa::ipu3 {
*/
/**
- * \struct IPAFrameContext
- * \brief Context for a frame
- *
- * The frame context stores data specific to a single frame processed by the
- * IPA. Each frame processed by the IPA has a context associated with it,
- * accessible through the IPAContext structure.
- *
- * Fields in the frame context should reflect values and controls
- * associated with the specific frame as requested by the application, and
- * as configured by the hardware. Fields can be read by algorithms to
- * determine if they should update any specific action for this frame, and
- * finally to update the metadata control lists when the frame is fully
- * completed.
- */
-
-/**
* \struct IPAContext
* \brief Global IPA context data shared between all algorithms
*
@@ -63,6 +47,9 @@ namespace libcamera::ipa::ipu3 {
*
* \var IPAContext::activeState
* \brief The current state of IPA algorithms
+ *
+ * \var IPAContext::ctrlMap
+ * \brief A ControlInfoMap::Map of controls populated by the algorithms
*/
/**
@@ -84,22 +71,21 @@ namespace libcamera::ipa::ipu3 {
* \brief AF grid configuration of the IPA
*
* \var IPASessionConfiguration::af.afGrid
- * \brief AF scene grid configuration.
+ * \brief AF scene grid configuration
*/
/**
* \var IPAActiveState::af
* \brief Context for the Automatic Focus algorithm
*
- * \struct IPAActiveState::af
* \var IPAActiveState::af.focus
* \brief Current position of the lens
*
* \var IPAActiveState::af.maxVariance
- * \brief The maximum variance of the current image.
+ * \brief The maximum variance of the current image
*
* \var IPAActiveState::af.stable
- * \brief It is set to true, if the best focus is found.
+ * \brief It is set to true, if the best focus is found
*/
/**
@@ -128,6 +114,9 @@ namespace libcamera::ipa::ipu3 {
*
* \var IPASessionConfiguration::sensor.defVBlank
* \brief The default vblank value of the sensor
+ *
+ * \var IPASessionConfiguration::sensor.size
+ * \brief Sensor output resolution
*/
/**
@@ -150,7 +139,7 @@ namespace libcamera::ipa::ipu3 {
* \var IPAActiveState::awb
* \brief Context for the Automatic White Balance algorithm
*
- * \struct IPAActiveState::awb.gains
+ * \var IPAActiveState::awb.gains
* \brief White balance gains
*
* \var IPAActiveState::awb.gains.red
@@ -181,25 +170,8 @@ namespace libcamera::ipa::ipu3 {
*/
/**
- * \brief Default constructor for IPAFrameContext
- */
-IPAFrameContext::IPAFrameContext() = default;
-
-/**
- * \brief Construct a IPAFrameContext instance
- */
-IPAFrameContext::IPAFrameContext(uint32_t id, const ControlList &reqControls)
- : frame(id), frameControls(reqControls)
-{
- sensor = {};
-}
-
-/**
- * \var IPAFrameContext::frame
- * \brief The frame number
- *
- * \var IPAFrameContext::frameControls
- * \brief Controls sent in by the application while queuing the request
+ * \struct IPAFrameContext
+ * \brief IPU3-specific FrameContext
*
* \var IPAFrameContext::sensor
* \brief Effective sensor values that were applied for the frame
diff --git a/src/ipa/ipu3/ipa_context.h b/src/ipa/ipu3/ipa_context.h
index 42e11141..c85d1e34 100644
--- a/src/ipa/ipu3/ipa_context.h
+++ b/src/ipa/ipu3/ipa_context.h
@@ -2,14 +2,12 @@
/*
* Copyright (C) 2021, Google Inc.
*
- * ipa_context.h - IPU3 IPA Context
+ * IPU3 IPA Context
*
*/
#pragma once
-#include <array>
-
#include <linux/intel-ipu3.h>
#include <libcamera/base/utils.h>
@@ -17,13 +15,12 @@
#include <libcamera/controls.h>
#include <libcamera/geometry.h>
+#include <libipa/fc_queue.h>
+
namespace libcamera {
namespace ipa::ipu3 {
-/* Maximum number of frame contexts to be held */
-static constexpr uint32_t kMaxFrameContexts = 16;
-
struct IPASessionConfiguration {
struct {
ipu3_uapi_grid_config bdsGrid;
@@ -45,6 +42,7 @@ struct IPASessionConfiguration {
struct {
int32_t defVBlank;
utils::Duration lineDuration;
+ Size size;
} sensor;
};
@@ -58,6 +56,8 @@ struct IPAActiveState {
struct {
uint32_t exposure;
double gain;
+ uint32_t constraintMode;
+ uint32_t exposureMode;
} agc;
struct {
@@ -76,24 +76,20 @@ struct IPAActiveState {
} toneMapping;
};
-struct IPAFrameContext {
- IPAFrameContext();
- IPAFrameContext(uint32_t id, const ControlList &reqControls);
-
+struct IPAFrameContext : public FrameContext {
struct {
uint32_t exposure;
double gain;
} sensor;
-
- uint32_t frame;
- ControlList frameControls;
};
struct IPAContext {
IPASessionConfiguration configuration;
IPAActiveState activeState;
- std::array<IPAFrameContext, kMaxFrameContexts> frameContexts;
+ FCQueue<IPAFrameContext> frameContexts;
+
+ ControlInfoMap::Map ctrlMap;
};
} /* namespace ipa::ipu3 */
diff --git a/src/ipa/ipu3/ipu3-ipa-design-guide.rst b/src/ipa/ipu3/ipu3-ipa-design-guide.rst
index e724fdda..72506397 100644
--- a/src/ipa/ipu3/ipu3-ipa-design-guide.rst
+++ b/src/ipa/ipu3/ipu3-ipa-design-guide.rst
@@ -1,3 +1,5 @@
+.. SPDX-License-Identifier: CC-BY-SA-4.0
+
IPU3 IPA Architecture Design and Overview
=========================================
diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp
index 2f6bb672..cdcdf1fb 100644
--- a/src/ipa/ipu3/ipu3.cpp
+++ b/src/ipa/ipu3/ipu3.cpp
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2020, Google Inc.
*
- * ipu3.cpp - IPU3 Image Processing Algorithms
+ * IPU3 Image Processing Algorithms
*/
#include <algorithm>
@@ -18,6 +18,7 @@
#include <linux/intel-ipu3.h>
#include <linux/v4l2-controls.h>
+#include <libcamera/base/file.h>
#include <libcamera/base/log.h>
#include <libcamera/base/utils.h>
@@ -29,6 +30,7 @@
#include <libcamera/request.h>
#include "libcamera/internal/mapped_framebuffer.h"
+#include "libcamera/internal/yaml_parser.h"
#include "algorithms/af.h"
#include "algorithms/agc.h"
@@ -38,6 +40,8 @@
#include "algorithms/tone_mapping.h"
#include "libipa/camera_sensor_helper.h"
+#include "ipa_context.h"
+
/* Minimum grid width, expressed as a number of cells */
static constexpr uint32_t kMinGridWidth = 16;
/* Maximum grid width, expressed as a number of cells */
@@ -51,6 +55,9 @@ static constexpr uint32_t kMinCellSizeLog2 = 3;
/* log2 of the maximum grid cell width and height, in pixels */
static constexpr uint32_t kMaxCellSizeLog2 = 6;
+/* Maximum number of frame contexts to be held */
+static constexpr uint32_t kMaxFrameContexts = 16;
+
namespace libcamera {
LOG_DEFINE_CATEGORY(IPAIPU3)
@@ -71,7 +78,7 @@ namespace ipa::ipu3 {
*
* At initialisation time, a CameraSensorHelper is instantiated to support
* camera-specific calculations, while the default controls are computed, and
- * the algorithms are constructed and placed in an ordered list.
+ * the algorithms are instantiated from the tuning data file.
*
* The IPU3 ImgU operates with a grid layout to divide the overall frame into
* rectangular cells of pixels. When the IPA is configured, we determine the
@@ -92,12 +99,14 @@ namespace ipa::ipu3 {
* fillParamsBuffer() call.
*
* The individual algorithms are split into modular components that are called
- * iteratively to allow them to process statistics from the ImgU in a defined
- * order.
+ * iteratively to allow them to process statistics from the ImgU in the order
+ * defined in the tuning data file.
*
- * The current implementation supports three core algorithms:
- * - Automatic white balance (AWB)
+ * The current implementation supports five core algorithms:
+ *
+ * - Auto focus (AF)
* - Automatic gain and exposure control (AGC)
+ * - Automatic white balance (AWB)
* - Black level correction (BLC)
* - Tone mapping (Gamma)
*
@@ -128,9 +137,11 @@ namespace ipa::ipu3 {
* sensor-specific tuning to adapt for Black Level compensation (BLC), Lens
* shading correction (SHD) and Color correction (CCM).
*/
-class IPAIPU3 : public IPAIPU3Interface
+class IPAIPU3 : public IPAIPU3Interface, public Module
{
public:
+ IPAIPU3();
+
int init(const IPASettings &settings,
const IPACameraSensorInfo &sensorInfo,
const ControlInfoMap &sensorControls,
@@ -150,14 +161,16 @@ public:
void processStatsBuffer(const uint32_t frame, const int64_t frameTimestamp,
const uint32_t bufferId,
const ControlList &sensorControls) override;
+
+protected:
+ std::string logPrefix() const override;
+
private:
void updateControls(const IPACameraSensorInfo &sensorInfo,
const ControlInfoMap &sensorControls,
ControlInfoMap *ipaControls);
void updateSessionConfiguration(const ControlInfoMap &sensorControls);
- bool validateSensorControls();
-
void setControls(unsigned int frame);
void calculateBdsGrid(const Size &bdsOutputSize);
@@ -171,13 +184,20 @@ private:
/* Interface to the Camera Helper */
std::unique_ptr<CameraSensorHelper> camHelper_;
- /* Maintain the algorithms used by the IPA */
- std::list<std::unique_ptr<ipa::ipu3::Algorithm>> algorithms_;
-
/* Local parameter storage */
struct IPAContext context_;
};
+IPAIPU3::IPAIPU3()
+ : context_({ {}, {}, { kMaxFrameContexts }, {} })
+{
+}
+
+std::string IPAIPU3::logPrefix() const
+{
+ return "ipu3";
+}
+
/**
* \brief Compute IPASessionConfiguration using the sensor information and the
* sensor V4L2 controls
@@ -267,32 +287,11 @@ void IPAIPU3::updateControls(const IPACameraSensorInfo &sensorInfo,
frameDurations[1],
frameDurations[2]);
+ controls.merge(context_.ctrlMap);
*ipaControls = ControlInfoMap(std::move(controls), controls::controls);
}
/**
- * \brief Validate that the sensor controls mandatory for the IPA exists
- */
-bool IPAIPU3::validateSensorControls()
-{
- static const uint32_t ctrls[] = {
- V4L2_CID_ANALOGUE_GAIN,
- V4L2_CID_EXPOSURE,
- V4L2_CID_VBLANK,
- };
-
- for (auto c : ctrls) {
- if (sensorCtrls_.find(c) == sensorCtrls_.end()) {
- LOG(IPAIPU3, Error) << "Unable to find sensor control "
- << utils::hex(c);
- return false;
- }
- }
-
- return true;
-}
-
-/**
* \brief Initialize the IPA module and its controls
*
* This function receives the camera sensor information from the pipeline
@@ -304,7 +303,7 @@ int IPAIPU3::init(const IPASettings &settings,
const ControlInfoMap &sensorControls,
ControlInfoMap *ipaControls)
{
- camHelper_ = CameraSensorHelperFactory::create(settings.sensorModel);
+ camHelper_ = CameraSensorHelperFactoryBase::create(settings.sensorModel);
if (camHelper_ == nullptr) {
LOG(IPAIPU3, Error)
<< "Failed to create camera sensor helper for "
@@ -314,14 +313,39 @@ int IPAIPU3::init(const IPASettings &settings,
/* Clean context */
context_.configuration = {};
- context_.configuration.sensor.lineDuration = sensorInfo.lineLength * 1.0s / sensorInfo.pixelRate;
+ context_.configuration.sensor.lineDuration = sensorInfo.minLineLength
+ * 1.0s / sensorInfo.pixelRate;
+
+ /* Load the tuning data file. */
+ File file(settings.configurationFile);
+ if (!file.open(File::OpenModeFlag::ReadOnly)) {
+ int ret = file.error();
+ LOG(IPAIPU3, Error)
+ << "Failed to open configuration file "
+ << settings.configurationFile << ": " << strerror(-ret);
+ return ret;
+ }
- /* Construct our Algorithms */
- algorithms_.push_back(std::make_unique<algorithms::Af>());
- algorithms_.push_back(std::make_unique<algorithms::Agc>());
- algorithms_.push_back(std::make_unique<algorithms::Awb>());
- algorithms_.push_back(std::make_unique<algorithms::BlackLevelCorrection>());
- algorithms_.push_back(std::make_unique<algorithms::ToneMapping>());
+ std::unique_ptr<libcamera::YamlObject> data = YamlParser::parse(file);
+ if (!data)
+ return -EINVAL;
+
+ unsigned int version = (*data)["version"].get<uint32_t>(0);
+ if (version != 1) {
+ LOG(IPAIPU3, Error)
+ << "Invalid tuning file version " << version;
+ return -EINVAL;
+ }
+
+ if (!data->contains("algorithms")) {
+ LOG(IPAIPU3, Error)
+ << "Tuning file doesn't contain any algorithm";
+ return -EINVAL;
+ }
+
+ int ret = createAlgorithms(context_, (*data)["algorithms"]);
+ if (ret)
+ return ret;
/* Initialize controls. */
updateControls(sensorInfo, sensorControls, ipaControls);
@@ -348,6 +372,7 @@ int IPAIPU3::start()
*/
void IPAIPU3::stop()
{
+ context_.frameContexts.clear();
}
/**
@@ -446,6 +471,16 @@ int IPAIPU3::configure(const IPAConfigInfo &configInfo,
lensCtrls_ = configInfo.lensControls;
+ /* Clear the IPA context for the new streaming session. */
+ context_.activeState = {};
+ context_.configuration = {};
+ context_.frameContexts.clear();
+
+ /* Initialise the sensor configuration. */
+ context_.configuration.sensor.lineDuration = sensorInfo_.minLineLength
+ * 1.0s / sensorInfo_.pixelRate;
+ context_.configuration.sensor.size = sensorInfo_.outputSize;
+
/*
* Compute the sensor V4L2 controls to be used by the algorithms and
* to be set on the sensor.
@@ -454,23 +489,13 @@ int IPAIPU3::configure(const IPAConfigInfo &configInfo,
calculateBdsGrid(configInfo.bdsOutputSize);
- /* Clean IPAActiveState at each reconfiguration. */
- context_.activeState = {};
- IPAFrameContext initFrameContext;
- context_.frameContexts.fill(initFrameContext);
-
- if (!validateSensorControls()) {
- LOG(IPAIPU3, Error) << "Sensor control validation failed.";
- return -EINVAL;
- }
-
/* Update the camera controls using the new sensor settings. */
updateControls(sensorInfo_, sensorCtrls_, ipaControls);
/* Update the IPASessionConfiguration using the sensor settings. */
updateSessionConfiguration(sensorCtrls_);
- for (auto const &algo : algorithms_) {
+ for (auto const &algo : algorithms()) {
int ret = algo->configure(context_, configInfo);
if (ret)
return ret;
@@ -538,8 +563,10 @@ void IPAIPU3::fillParamsBuffer(const uint32_t frame, const uint32_t bufferId)
*/
params->use = {};
- for (auto const &algo : algorithms_)
- algo->prepare(context_, params);
+ IPAFrameContext &frameContext = context_.frameContexts.get(frame);
+
+ for (auto const &algo : algorithms())
+ algo->prepare(context_, frame, frameContext, params);
paramsBufferReady.emit(frame);
}
@@ -569,33 +596,18 @@ void IPAIPU3::processStatsBuffer(const uint32_t frame,
const ipu3_uapi_stats_3a *stats =
reinterpret_cast<ipu3_uapi_stats_3a *>(mem.data());
- IPAFrameContext &frameContext = context_.frameContexts[frame % kMaxFrameContexts];
-
- if (frameContext.frame != frame)
- LOG(IPAIPU3, Warning) << "Frame " << frame << " does not match its frame context";
+ IPAFrameContext &frameContext = context_.frameContexts.get(frame);
frameContext.sensor.exposure = sensorControls.get(V4L2_CID_EXPOSURE).get<int32_t>();
frameContext.sensor.gain = camHelper_->gain(sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get<int32_t>());
- double lineDuration = context_.configuration.sensor.lineDuration.get<std::micro>();
- int32_t vBlank = context_.configuration.sensor.defVBlank;
- ControlList ctrls(controls::controls);
+ ControlList metadata(controls::controls);
- for (auto const &algo : algorithms_)
- algo->process(context_, &frameContext, stats);
+ for (auto const &algo : algorithms())
+ algo->process(context_, frame, frameContext, stats, metadata);
setControls(frame);
- /* \todo Use VBlank value calculated from each frame exposure. */
- int64_t frameDuration = (vBlank + sensorInfo_.outputSize.height) * lineDuration;
- ctrls.set(controls::FrameDuration, frameDuration);
-
- ctrls.set(controls::AnalogueGain, frameContext.sensor.gain);
-
- ctrls.set(controls::ColourTemperature, context_.activeState.awb.temperatureK);
-
- ctrls.set(controls::ExposureTime, frameContext.sensor.exposure * lineDuration);
-
/*
* \todo The Metadata provides a path to getting extended data
* out to the application. Further data such as a simplifed Histogram
@@ -604,7 +616,7 @@ void IPAIPU3::processStatsBuffer(const uint32_t frame,
* likely want to avoid putting platform specific metadata in.
*/
- metadataReady.emit(frame, ctrls);
+ metadataReady.emit(frame, metadata);
}
/**
@@ -617,8 +629,10 @@ void IPAIPU3::processStatsBuffer(const uint32_t frame,
*/
void IPAIPU3::queueRequest(const uint32_t frame, const ControlList &controls)
{
- /* \todo Start processing for 'frame' based on 'controls'. */
- context_.frameContexts[frame % kMaxFrameContexts] = { frame, controls };
+ IPAFrameContext &frameContext = context_.frameContexts.alloc(frame);
+
+ for (auto const &algo : algorithms())
+ algo->queueRequest(context_, frame, frameContext, controls);
}
/**
@@ -659,7 +673,7 @@ extern "C" {
const struct IPAModuleInfo ipaModuleInfo = {
IPA_MODULE_API_VERSION,
1,
- "PipelineHandlerIPU3",
+ "ipu3",
"ipu3",
};
diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build
index 3194111a..66c39843 100644
--- a/src/ipa/ipu3/meson.build
+++ b/src/ipa/ipu3/meson.build
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: CC0-1.0
subdir('algorithms')
+subdir('data')
ipa_name = 'ipa_ipu3'
@@ -28,3 +29,5 @@ if ipa_sign_module
install : false,
build_by_default : true)
endif
+
+ipa_names += ipa_name
diff --git a/src/ipa/ipu3/module.h b/src/ipa/ipu3/module.h
index d94fc459..60f65cc4 100644
--- a/src/ipa/ipu3/module.h
+++ b/src/ipa/ipu3/module.h
@@ -2,7 +2,7 @@
/*
* Copyright (C) 2022, Ideas On Board
*
- * module.h - IPU3 IPA Module
+ * IPU3 IPA Module
*/
#pragma once
generated by cgit v1.2.1 (git 2.18.0) at 2024-05-19 04:35:27 +0000