From 44cffefd60822181851efbd8b1372988787b8119 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Tue, 15 Nov 2022 09:07:53 +0000 Subject: ipa: raspberrypi: Use an array of RPiController::Metadata objects Allow the IPA to cycle through an array of RPiController::Metadata objects, one per prepare()/process() invocation, when running the controller algorithms. This allows historical metadata objects to be retained, and subsequently passed into the controller algorithms on future frames. At present, only a single index into this array is used. This change provides a route to fixing a problem with the AGC algorithm, where if a manual shutter/gain is requested, the algorithm does not currently retain any context of the total exposure that it has calculated. As a result, the wrong digital gain would be applied when the frame with the manual shutter/gain is processed by the ISP. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Tested-by: David Plowman Reviewed-by: Kieran Bingham Signed-off-by: Laurent Pinchart --- src/ipa/raspberrypi/raspberrypi.cpp | 63 ++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/src/ipa/raspberrypi/raspberrypi.cpp b/src/ipa/raspberrypi/raspberrypi.cpp index 4e10c57d..ea3e90a3 100644 --- a/src/ipa/raspberrypi/raspberrypi.cpp +++ b/src/ipa/raspberrypi/raspberrypi.cpp @@ -57,6 +57,9 @@ namespace libcamera { using namespace std::literals::chrono_literals; using utils::Duration; +/* Number of metadata objects available in the context list. */ +constexpr unsigned int numMetadataContexts = 16; + /* Configure the sensor with these values initially. */ constexpr double defaultAnalogueGain = 1.0; constexpr Duration defaultExposureTime = 20.0ms; @@ -163,7 +166,7 @@ private: /* Raspberry Pi controller specific defines. */ std::unique_ptr helper_; RPiController::Controller controller_; - RPiController::Metadata rpiMetadata_; + std::array rpiMetadata_; /* * We count frames to decide if the frame must be hidden (e.g. from @@ -539,14 +542,15 @@ void IPARPi::signalIspPrepare(const ISPConfig &data) void IPARPi::reportMetadata() { - std::unique_lock lock(rpiMetadata_); + RPiController::Metadata &rpiMetadata = rpiMetadata_[0]; + std::unique_lock lock(rpiMetadata); /* * Certain information about the current frame and how it will be * processed can be extracted and placed into the libcamera metadata * buffer, where an application could query it. */ - DeviceStatus *deviceStatus = rpiMetadata_.getLocked("device.status"); + DeviceStatus *deviceStatus = rpiMetadata.getLocked("device.status"); if (deviceStatus) { libcameraMetadata_.set(controls::ExposureTime, deviceStatus->shutterSpeed.get()); @@ -557,24 +561,24 @@ void IPARPi::reportMetadata() libcameraMetadata_.set(controls::SensorTemperature, *deviceStatus->sensorTemperature); } - AgcStatus *agcStatus = rpiMetadata_.getLocked("agc.status"); + AgcStatus *agcStatus = rpiMetadata.getLocked("agc.status"); if (agcStatus) { libcameraMetadata_.set(controls::AeLocked, agcStatus->locked); libcameraMetadata_.set(controls::DigitalGain, agcStatus->digitalGain); } - LuxStatus *luxStatus = rpiMetadata_.getLocked("lux.status"); + LuxStatus *luxStatus = rpiMetadata.getLocked("lux.status"); if (luxStatus) libcameraMetadata_.set(controls::Lux, luxStatus->lux); - AwbStatus *awbStatus = rpiMetadata_.getLocked("awb.status"); + AwbStatus *awbStatus = rpiMetadata.getLocked("awb.status"); if (awbStatus) { libcameraMetadata_.set(controls::ColourGains, { static_cast(awbStatus->gainR), static_cast(awbStatus->gainB) }); libcameraMetadata_.set(controls::ColourTemperature, awbStatus->temperatureK); } - BlackLevelStatus *blackLevelStatus = rpiMetadata_.getLocked("black_level.status"); + BlackLevelStatus *blackLevelStatus = rpiMetadata.getLocked("black_level.status"); if (blackLevelStatus) libcameraMetadata_.set(controls::SensorBlackLevels, { static_cast(blackLevelStatus->blackLevelR), @@ -582,7 +586,7 @@ void IPARPi::reportMetadata() static_cast(blackLevelStatus->blackLevelG), static_cast(blackLevelStatus->blackLevelB) }); - FocusStatus *focusStatus = rpiMetadata_.getLocked("focus.status"); + FocusStatus *focusStatus = rpiMetadata.getLocked("focus.status"); if (focusStatus && focusStatus->num == 12) { /* * We get a 4x3 grid of regions by default. Calculate the average @@ -593,7 +597,7 @@ void IPARPi::reportMetadata() libcameraMetadata_.set(controls::FocusFoM, focusFoM); } - CcmStatus *ccmStatus = rpiMetadata_.getLocked("ccm.status"); + CcmStatus *ccmStatus = rpiMetadata.getLocked("ccm.status"); if (ccmStatus) { float m[9]; for (unsigned int i = 0; i < 9; i++) @@ -1008,9 +1012,10 @@ void IPARPi::prepareISP(const ISPConfig &data) { int64_t frameTimestamp = data.controls.get(controls::SensorTimestamp).value_or(0); RPiController::Metadata lastMetadata; + RPiController::Metadata &rpiMetadata = rpiMetadata_[0]; Span embeddedBuffer; - lastMetadata = std::move(rpiMetadata_); + lastMetadata = std::move(rpiMetadata); fillDeviceStatus(data.controls); if (data.embeddedBufferPresent) { @@ -1027,7 +1032,7 @@ void IPARPi::prepareISP(const ISPConfig &data) * This may overwrite the DeviceStatus using values from the sensor * metadata, and may also do additional custom processing. */ - helper_->prepare(embeddedBuffer, rpiMetadata_); + helper_->prepare(embeddedBuffer, rpiMetadata); /* Done with embedded data now, return to pipeline handler asap. */ if (data.embeddedBufferPresent) @@ -1043,7 +1048,7 @@ void IPARPi::prepareISP(const ISPConfig &data) * current frame, or any other bits of metadata that were added * in helper_->Prepare(). */ - rpiMetadata_.merge(lastMetadata); + rpiMetadata.merge(lastMetadata); processPending_ = false; return; } @@ -1053,48 +1058,48 @@ void IPARPi::prepareISP(const ISPConfig &data) ControlList ctrls(ispCtrls_); - controller_.prepare(&rpiMetadata_); + controller_.prepare(&rpiMetadata); /* Lock the metadata buffer to avoid constant locks/unlocks. */ - std::unique_lock lock(rpiMetadata_); + std::unique_lock lock(rpiMetadata); - AwbStatus *awbStatus = rpiMetadata_.getLocked("awb.status"); + AwbStatus *awbStatus = rpiMetadata.getLocked("awb.status"); if (awbStatus) applyAWB(awbStatus, ctrls); - CcmStatus *ccmStatus = rpiMetadata_.getLocked("ccm.status"); + CcmStatus *ccmStatus = rpiMetadata.getLocked("ccm.status"); if (ccmStatus) applyCCM(ccmStatus, ctrls); - AgcStatus *dgStatus = rpiMetadata_.getLocked("agc.status"); + AgcStatus *dgStatus = rpiMetadata.getLocked("agc.status"); if (dgStatus) applyDG(dgStatus, ctrls); - AlscStatus *lsStatus = rpiMetadata_.getLocked("alsc.status"); + AlscStatus *lsStatus = rpiMetadata.getLocked("alsc.status"); if (lsStatus) applyLS(lsStatus, ctrls); - ContrastStatus *contrastStatus = rpiMetadata_.getLocked("contrast.status"); + ContrastStatus *contrastStatus = rpiMetadata.getLocked("contrast.status"); if (contrastStatus) applyGamma(contrastStatus, ctrls); - BlackLevelStatus *blackLevelStatus = rpiMetadata_.getLocked("black_level.status"); + BlackLevelStatus *blackLevelStatus = rpiMetadata.getLocked("black_level.status"); if (blackLevelStatus) applyBlackLevel(blackLevelStatus, ctrls); - GeqStatus *geqStatus = rpiMetadata_.getLocked("geq.status"); + GeqStatus *geqStatus = rpiMetadata.getLocked("geq.status"); if (geqStatus) applyGEQ(geqStatus, ctrls); - DenoiseStatus *denoiseStatus = rpiMetadata_.getLocked("denoise.status"); + DenoiseStatus *denoiseStatus = rpiMetadata.getLocked("denoise.status"); if (denoiseStatus) applyDenoise(denoiseStatus, ctrls); - SharpenStatus *sharpenStatus = rpiMetadata_.getLocked("sharpen.status"); + SharpenStatus *sharpenStatus = rpiMetadata.getLocked("sharpen.status"); if (sharpenStatus) applySharpen(sharpenStatus, ctrls); - DpcStatus *dpcStatus = rpiMetadata_.getLocked("dpc.status"); + DpcStatus *dpcStatus = rpiMetadata.getLocked("dpc.status"); if (dpcStatus) applyDPC(dpcStatus, ctrls); @@ -1118,11 +1123,13 @@ void IPARPi::fillDeviceStatus(const ControlList &sensorControls) LOG(IPARPI, Debug) << "Metadata - " << deviceStatus; - rpiMetadata_.set("device.status", deviceStatus); + rpiMetadata_[0].set("device.status", deviceStatus); } void IPARPi::processStats(unsigned int bufferId) { + RPiController::Metadata &rpiMetadata = rpiMetadata_[0]; + auto it = buffers_.find(bufferId); if (it == buffers_.end()) { LOG(IPARPI, Error) << "Could not find stats buffer!"; @@ -1132,11 +1139,11 @@ void IPARPi::processStats(unsigned int bufferId) Span mem = it->second.planes()[0]; bcm2835_isp_stats *stats = reinterpret_cast(mem.data()); RPiController::StatisticsPtr statistics = std::make_shared(*stats); - helper_->process(statistics, rpiMetadata_); - controller_.process(statistics, &rpiMetadata_); + helper_->process(statistics, rpiMetadata); + controller_.process(statistics, &rpiMetadata); struct AgcStatus agcStatus; - if (rpiMetadata_.get("agc.status", agcStatus) == 0) { + if (rpiMetadata.get("agc.status", agcStatus) == 0) { ControlList ctrls(sensorCtrls_); applyAGC(&agcStatus, ctrls); -- cgit v1.2.1