ipa: rpi: Add HDR support

Add support for the following HDR modes in the Raspberry Pi IPA:
- Night mode
- Single exposure mode
- Multi-exposure (merged and unmerged)

The algorithm is updated to expect the HDR short channel to meter
explicitly for highlights. This means that it will not in general
under-expose the short channel more than is actually necessary.

When images don't have much saturation, it's good to detect this so
that some of the boost we want to apply to the dark areas can be
implemented as regular gain. This means we can then adjust the tone
curve less, leading to less flat looking images.

The impact on the HDR algorithm is then that this determines how we
build tonemaps dynamically. The highlights are more-or-less correct
now, so we have to build a power-type curve that gives us the
appropriately configured targets in the lower part of the histogram.

We allow the tuning file to supply the maximum spatial gain value,
rather than the whole curve (though it can supply this if it
wants). Some parameter defaults are tweaked to be generally better
across the range of our cameras.

Signed-off-by: David Plowman <david.plowman@raspberrypi.com>
Signed-off-by: Naushir Patuck <naush@raspberrypi.com>
Reviewed-by: David Plowman <david.plowman@raspberrypi.com>
Reviewed-by: Naushir Patuck <naush@raspberrypi.com>

1 files changed, 107 insertions, 4 deletions

diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp
index a1fec3aa..97a32522 100644
--- a/src/ipa/rpi/common/ipa_base.cpp
+++ b/src/ipa/rpi/common/ipa_base.cpp
@@ -24,6 +24,7 @@
 #include "controller/ccm_status.h"
 #include "controller/contrast_algorithm.h"
 #include "controller/denoise_algorithm.h"
+#include "controller/hdr_algorithm.h"
 #include "controller/lux_status.h"
 #include "controller/sharpen_algorithm.h"
 #include "controller/statistics.h"
@@ -67,6 +68,7 @@ const ControlInfoMap::Map ipaControls{
 	{ &controls::AeFlickerPeriod, ControlInfo(100, 1000000) },
 	{ &controls::Brightness, ControlInfo(-1.0f, 1.0f, 0.0f) },
 	{ &controls::Contrast, ControlInfo(0.0f, 32.0f, 1.0f) },
+	{ &controls::HdrMode, ControlInfo(controls::HdrModeValues) },
 	{ &controls::Sharpness, ControlInfo(0.0f, 16.0f, 1.0f) },
 	{ &controls::ScalerCrop, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) },
 	{ &controls::FrameDurationLimits, ControlInfo(INT64_C(33333), INT64_C(120000)) },
@@ -101,7 +103,8 @@ namespace ipa::RPi {
 
 IpaBase::IpaBase()
 	: controller_(), frameLengths_(FrameLengthsQueueSize, 0s), frameCount_(0),
-	  mistrustCount_(0), lastRunTimestamp_(0), firstStart_(true), flickerState_({ 0, 0s })
+	  mistrustCount_(0), lastRunTimestamp_(0), firstStart_(true), flickerState_({ 0, 0s }),
+	  stitchSwapBuffers_(false)
 {
 }
 
@@ -294,6 +297,8 @@ void IpaBase::start(const ControlList &controls, StartResult *result)
 		result->controls = std::move(ctrls);
 		setCameraTimeoutValue();
 	}
+	/* Make a note of this as it tells us the HDR status of the first few frames. */
+	hdrStatus_ = agcStatus.hdr;
 
 	/*
 	 * Initialise frame counts, and decide how many frames must be hidden or
@@ -397,11 +402,17 @@ void IpaBase::prepareIsp(const PrepareParams &params)
 	 * sensor exposure/gain changes. So fetch it from the metadata list
 	 * indexed by the IPA cookie returned, and put it in the current frame
 	 * metadata.
+	 *
+	 * Note if the HDR mode has changed, as things like tonemaps may need updating.
 	 */
 	AgcStatus agcStatus;
+	bool hdrChange = false;
 	RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext];
-	if (!delayedMetadata.get<AgcStatus>("agc.status", agcStatus))
+	if (!delayedMetadata.get<AgcStatus>("agc.status", agcStatus)) {
 		rpiMetadata.set("agc.delayed_status", agcStatus);
+		hdrChange = agcStatus.hdr.mode != hdrStatus_.mode;
+		hdrStatus_ = agcStatus.hdr;
+	}
 
 	/*
 	 * This may overwrite the DeviceStatus using values from the sensor
@@ -412,7 +423,7 @@ void IpaBase::prepareIsp(const PrepareParams &params)
 	/* Allow a 10% margin on the comparison below. */
 	Duration delta = (frameTimestamp - lastRunTimestamp_) * 1.0ns;
 	if (lastRunTimestamp_ && frameCount_ > dropFrameCount_ &&
-	    delta < controllerMinFrameDuration * 0.9) {
+	    delta < controllerMinFrameDuration * 0.9 && !hdrChange) {
 		/*
 		 * Ensure we merge the previous frame's metadata with the current
 		 * frame. This will not overwrite exposure/gain values for the
@@ -449,7 +460,7 @@ void IpaBase::prepareIsp(const PrepareParams &params)
 		reportMetadata(ipaContext);
 
 	/* Ready to push the input buffer into the ISP. */
-	prepareIspComplete.emit(params.buffers, false);
+	prepareIspComplete.emit(params.buffers, stitchSwapBuffers_);
 }
 
 void IpaBase::processStats(const ProcessParams &params)
@@ -661,9 +672,21 @@ static const std::map<int32_t, RPiController::AfAlgorithm::AfPause> AfPauseTable
 	{ controls::AfPauseResume, RPiController::AfAlgorithm::AfPauseResume },
 };
 
+static const std::map<int32_t, std::string> HdrModeTable = {
+	{ controls::HdrModeOff, "Off" },
+	{ controls::HdrModeMultiExposureUnmerged, "MultiExposureUnmerged" },
+	{ controls::HdrModeMultiExposure, "MultiExposure" },
+	{ controls::HdrModeSingleExposure, "SingleExposure" },
+	{ controls::HdrModeNight, "Night" },
+};
+
 void IpaBase::applyControls(const ControlList &controls)
 {
+	using RPiController::AgcAlgorithm;
 	using RPiController::AfAlgorithm;
+	using RPiController::ContrastAlgorithm;
+	using RPiController::DenoiseAlgorithm;
+	using RPiController::HdrAlgorithm;
 
 	/* Clear the return metadata buffer. */
 	libcameraMetadata_.clear();
@@ -1135,6 +1158,57 @@ void IpaBase::applyControls(const ControlList &controls)
 			break;
 		}
 
+		case controls::HDR_MODE: {
+			HdrAlgorithm *hdr = dynamic_cast<HdrAlgorithm *>(controller_.getAlgorithm("hdr"));
+			if (!hdr) {
+				LOG(IPARPI, Warning) << "No HDR algorithm available";
+				break;
+			}
+
+			auto mode = HdrModeTable.find(ctrl.second.get<int32_t>());
+			if (mode == HdrModeTable.end()) {
+				LOG(IPARPI, Warning) << "Unrecognised HDR mode";
+				break;
+			}
+
+			AgcAlgorithm *agc = dynamic_cast<AgcAlgorithm *>(controller_.getAlgorithm("agc"));
+			if (!agc) {
+				LOG(IPARPI, Warning) << "HDR requires an AGC algorithm";
+				break;
+			}
+
+			if (hdr->setMode(mode->second) == 0) {
+				agc->setActiveChannels(hdr->getChannels());
+
+				/* We also disable adpative contrast enhancement if HDR is running. */
+				ContrastAlgorithm *contrast =
+					dynamic_cast<ContrastAlgorithm *>(controller_.getAlgorithm("contrast"));
+				if (contrast) {
+					if (mode->second == "Off")
+						contrast->restoreCe();
+					else
+						contrast->enableCe(false);
+				}
+
+				DenoiseAlgorithm *denoise =
+					dynamic_cast<DenoiseAlgorithm *>(controller_.getAlgorithm("denoise"));
+				if (denoise) {
+					/* \todo - make the HDR mode say what denoise it wants? */
+					if (mode->second == "Night")
+						denoise->setConfig("night");
+					else if (mode->second == "SingleExposure")
+						denoise->setConfig("hdr");
+					/* MultiExposure doesn't need extra extra denoise. */
+					else
+						denoise->setConfig("normal");
+				}
+			} else
+				LOG(IPARPI, Warning)
+					<< "HDR mode " << mode->second << " not supported";
+
+			break;
+		}
+
 		default:
 			LOG(IPARPI, Warning)
 				<< "Ctrl " << controls::controls.at(ctrl.first)->name()
@@ -1282,6 +1356,35 @@ void IpaBase::reportMetadata(unsigned int ipaContext)
 		libcameraMetadata_.set(controls::AfPauseState, p);
 	}
 
+	/*
+	 * THe HDR algorithm sets the HDR channel into the agc.status at the time that those
+	 * AGC parameters were calculated several frames ago, so it comes back to us now in
+	 * the delayed_status. If this frame is too soon after a mode switch for the
+	 * delayed_status to be available, we use the HDR status that came out of the
+	 * switchMode call.
+	 */
+	const AgcStatus *agcStatus = rpiMetadata.getLocked<AgcStatus>("agc.delayed_status");
+	const HdrStatus &hdrStatus = agcStatus ? agcStatus->hdr : hdrStatus_;
+	if (!hdrStatus.mode.empty() && hdrStatus.mode != "Off") {
+		int32_t hdrMode = controls::HdrModeOff;
+		for (auto const &[mode, name] : HdrModeTable) {
+			if (hdrStatus.mode == name) {
+				hdrMode = mode;
+				break;
+			}
+		}
+		libcameraMetadata_.set(controls::HdrMode, hdrMode);
+
+		if (hdrStatus.channel == "short")
+			libcameraMetadata_.set(controls::HdrChannel, controls::HdrChannelShort);
+		else if (hdrStatus.channel == "long")
+			libcameraMetadata_.set(controls::HdrChannel, controls::HdrChannelLong);
+		else if (hdrStatus.channel == "medium")
+			libcameraMetadata_.set(controls::HdrChannel, controls::HdrChannelMedium);
+		else
+			libcameraMetadata_.set(controls::HdrChannel, controls::HdrChannelNone);
+	}
+
 	metadataReady.emit(libcameraMetadata_);
 }