From 726e9274ea95fa46352556d340c5793a8da51fcd Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Wed, 3 May 2023 13:20:27 +0100 Subject: pipeline: ipa: raspberrypi: Refactor and move the Raspberry Pi code Split the Raspberry Pi pipeline handler and IPA source code into common and VC4/BCM2835 specific file structures. For the pipeline handler, the common code files now live in src/libcamera/pipeline/rpi/common/ and the VC4-specific files in src/libcamera/pipeline/rpi/vc4/. For the IPA, the common code files now live in src/ipa/rpi/{cam_helper,controller}/ and the vc4 specific files in src/ipa/rpi/vc4/. With this change, the camera tuning files are now installed under share/libcamera/ipa/rpi/vc4/. To build the pipeline and IPA, the meson configuration options have now changed from "raspberrypi" to "rpi/vc4": meson setup build -Dipas=rpi/vc4 -Dpipelines=rpi/vc4 Signed-off-by: Naushir Patuck Reviewed-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- src/ipa/rpi/cam_helper/cam_helper.cpp | 265 ++++++++++++++++++++ src/ipa/rpi/cam_helper/cam_helper.h | 132 ++++++++++ src/ipa/rpi/cam_helper/cam_helper_imx219.cpp | 115 +++++++++ src/ipa/rpi/cam_helper/cam_helper_imx290.cpp | 68 +++++ src/ipa/rpi/cam_helper/cam_helper_imx296.cpp | 83 +++++++ src/ipa/rpi/cam_helper/cam_helper_imx477.cpp | 197 +++++++++++++++ src/ipa/rpi/cam_helper/cam_helper_imx519.cpp | 196 +++++++++++++++ src/ipa/rpi/cam_helper/cam_helper_imx708.cpp | 359 +++++++++++++++++++++++++++ src/ipa/rpi/cam_helper/cam_helper_ov5647.cpp | 109 ++++++++ src/ipa/rpi/cam_helper/cam_helper_ov9281.cpp | 66 +++++ src/ipa/rpi/cam_helper/md_parser.h | 155 ++++++++++++ src/ipa/rpi/cam_helper/md_parser_smia.cpp | 149 +++++++++++ src/ipa/rpi/cam_helper/meson.build | 26 ++ 13 files changed, 1920 insertions(+) create mode 100644 src/ipa/rpi/cam_helper/cam_helper.cpp create mode 100644 src/ipa/rpi/cam_helper/cam_helper.h create mode 100644 src/ipa/rpi/cam_helper/cam_helper_imx219.cpp create mode 100644 src/ipa/rpi/cam_helper/cam_helper_imx290.cpp create mode 100644 src/ipa/rpi/cam_helper/cam_helper_imx296.cpp create mode 100644 src/ipa/rpi/cam_helper/cam_helper_imx477.cpp create mode 100644 src/ipa/rpi/cam_helper/cam_helper_imx519.cpp create mode 100644 src/ipa/rpi/cam_helper/cam_helper_imx708.cpp create mode 100644 src/ipa/rpi/cam_helper/cam_helper_ov5647.cpp create mode 100644 src/ipa/rpi/cam_helper/cam_helper_ov9281.cpp create mode 100644 src/ipa/rpi/cam_helper/md_parser.h create mode 100644 src/ipa/rpi/cam_helper/md_parser_smia.cpp create mode 100644 src/ipa/rpi/cam_helper/meson.build (limited to 'src/ipa/rpi/cam_helper') diff --git a/src/ipa/rpi/cam_helper/cam_helper.cpp b/src/ipa/rpi/cam_helper/cam_helper.cpp new file mode 100644 index 00000000..ddd5e9a4 --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper.cpp @@ -0,0 +1,265 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi Ltd + * + * cam_helper.cpp - helper information for different sensors + */ + +#include + +#include +#include +#include + +#include "libcamera/internal/v4l2_videodevice.h" + +#include "cam_helper.h" +#include "md_parser.h" + +using namespace RPiController; +using namespace libcamera; +using libcamera::utils::Duration; +using namespace std::literals::chrono_literals; + +namespace libcamera { +LOG_DECLARE_CATEGORY(IPARPI) +} + +namespace { + +std::map &camHelpers() +{ + static std::map helpers; + return helpers; +} + +} /* namespace */ + +CamHelper *CamHelper::create(std::string const &camName) +{ + /* + * CamHelpers get registered by static RegisterCamHelper + * initialisers. + */ + for (auto &p : camHelpers()) { + if (camName.find(p.first) != std::string::npos) + return p.second(); + } + + return nullptr; +} + +CamHelper::CamHelper(std::unique_ptr parser, unsigned int frameIntegrationDiff) + : parser_(std::move(parser)), frameIntegrationDiff_(frameIntegrationDiff) +{ +} + +CamHelper::~CamHelper() +{ +} + +void CamHelper::prepare(Span buffer, + Metadata &metadata) +{ + parseEmbeddedData(buffer, metadata); +} + +void CamHelper::process([[maybe_unused]] StatisticsPtr &stats, + [[maybe_unused]] Metadata &metadata) +{ +} + +uint32_t CamHelper::exposureLines(const Duration exposure, const Duration lineLength) const +{ + return exposure / lineLength; +} + +Duration CamHelper::exposure(uint32_t exposureLines, const Duration lineLength) const +{ + return exposureLines * lineLength; +} + +std::pair CamHelper::getBlanking(Duration &exposure, + Duration minFrameDuration, + Duration maxFrameDuration) const +{ + uint32_t frameLengthMin, frameLengthMax, vblank, hblank; + Duration lineLength = mode_.minLineLength; + + /* + * minFrameDuration and maxFrameDuration are clamped by the caller + * based on the limits for the active sensor mode. + * + * frameLengthMax gets calculated on the smallest line length as we do + * not want to extend that unless absolutely necessary. + */ + frameLengthMin = minFrameDuration / mode_.minLineLength; + frameLengthMax = maxFrameDuration / mode_.minLineLength; + + /* + * Watch out for (exposureLines + frameIntegrationDiff_) overflowing a + * uint32_t in the std::clamp() below when the exposure time is + * extremely (extremely!) long - as happens when the IPA calculates the + * maximum possible exposure time. + */ + uint32_t exposureLines = std::min(CamHelper::exposureLines(exposure, lineLength), + std::numeric_limits::max() - frameIntegrationDiff_); + uint32_t frameLengthLines = std::clamp(exposureLines + frameIntegrationDiff_, + frameLengthMin, frameLengthMax); + + /* + * If our frame length lines is above the maximum allowed, see if we can + * extend the line length to accommodate the requested frame length. + */ + if (frameLengthLines > mode_.maxFrameLength) { + Duration lineLengthAdjusted = lineLength * frameLengthLines / mode_.maxFrameLength; + lineLength = std::min(mode_.maxLineLength, lineLengthAdjusted); + frameLengthLines = mode_.maxFrameLength; + } + + hblank = lineLengthToHblank(lineLength); + vblank = frameLengthLines - mode_.height; + + /* + * Limit the exposure to the maximum frame duration requested, and + * re-calculate if it has been clipped. + */ + exposureLines = std::min(frameLengthLines - frameIntegrationDiff_, + CamHelper::exposureLines(exposure, lineLength)); + exposure = CamHelper::exposure(exposureLines, lineLength); + + return { vblank, hblank }; +} + +Duration CamHelper::hblankToLineLength(uint32_t hblank) const +{ + return (mode_.width + hblank) * (1.0s / mode_.pixelRate); +} + +uint32_t CamHelper::lineLengthToHblank(const Duration &lineLength) const +{ + return (lineLength * mode_.pixelRate / 1.0s) - mode_.width; +} + +Duration CamHelper::lineLengthPckToDuration(uint32_t lineLengthPck) const +{ + return lineLengthPck * (1.0s / mode_.pixelRate); +} + +void CamHelper::setCameraMode(const CameraMode &mode) +{ + mode_ = mode; + if (parser_) { + parser_->reset(); + parser_->setBitsPerPixel(mode.bitdepth); + parser_->setLineLengthBytes(0); /* We use SetBufferSize. */ + } +} + +void CamHelper::getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const +{ + /* + * These values are correct for many sensors. Other sensors will + * need to over-ride this function. + */ + exposureDelay = 2; + gainDelay = 1; + vblankDelay = 2; + hblankDelay = 2; +} + +bool CamHelper::sensorEmbeddedDataPresent() const +{ + return false; +} + +double CamHelper::getModeSensitivity([[maybe_unused]] const CameraMode &mode) const +{ + /* + * Most sensors have the same sensitivity in every mode, but this + * function can be overridden for those that do not. Note that it is + * called before mode_ is set, so it must return the sensitivity + * of the mode that is passed in. + */ + return 1.0; +} + +unsigned int CamHelper::hideFramesStartup() const +{ + /* + * The number of frames when a camera first starts that shouldn't be + * displayed as they are invalid in some way. + */ + return 0; +} + +unsigned int CamHelper::hideFramesModeSwitch() const +{ + /* After a mode switch, many sensors return valid frames immediately. */ + return 0; +} + +unsigned int CamHelper::mistrustFramesStartup() const +{ + /* Many sensors return a single bad frame on start-up. */ + return 1; +} + +unsigned int CamHelper::mistrustFramesModeSwitch() const +{ + /* Many sensors return valid metadata immediately. */ + return 0; +} + +void CamHelper::parseEmbeddedData(Span buffer, + Metadata &metadata) +{ + MdParser::RegisterMap registers; + Metadata parsedMetadata; + + if (buffer.empty()) + return; + + if (parser_->parse(buffer, registers) != MdParser::Status::OK) { + LOG(IPARPI, Error) << "Embedded data buffer parsing failed"; + return; + } + + populateMetadata(registers, parsedMetadata); + metadata.merge(parsedMetadata); + + /* + * Overwrite the exposure/gain, line/frame length and sensor temperature values + * in the existing DeviceStatus with values from the parsed embedded buffer. + * Fetch it first in case any other fields were set meaningfully. + */ + DeviceStatus deviceStatus, parsedDeviceStatus; + if (metadata.get("device.status", deviceStatus) || + parsedMetadata.get("device.status", parsedDeviceStatus)) { + LOG(IPARPI, Error) << "DeviceStatus not found"; + return; + } + + deviceStatus.shutterSpeed = parsedDeviceStatus.shutterSpeed; + deviceStatus.analogueGain = parsedDeviceStatus.analogueGain; + deviceStatus.frameLength = parsedDeviceStatus.frameLength; + deviceStatus.lineLength = parsedDeviceStatus.lineLength; + if (parsedDeviceStatus.sensorTemperature) + deviceStatus.sensorTemperature = parsedDeviceStatus.sensorTemperature; + + LOG(IPARPI, Debug) << "Metadata updated - " << deviceStatus; + + metadata.set("device.status", deviceStatus); +} + +void CamHelper::populateMetadata([[maybe_unused]] const MdParser::RegisterMap ®isters, + [[maybe_unused]] Metadata &metadata) const +{ +} + +RegisterCamHelper::RegisterCamHelper(char const *camName, + CamHelperCreateFunc createFunc) +{ + camHelpers()[std::string(camName)] = createFunc; +} diff --git a/src/ipa/rpi/cam_helper/cam_helper.h b/src/ipa/rpi/cam_helper/cam_helper.h new file mode 100644 index 00000000..58a4b202 --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi Ltd + * + * cam_helper.h - helper class providing camera information + */ +#pragma once + +#include +#include +#include + +#include +#include + +#include "controller/camera_mode.h" +#include "controller/controller.h" +#include "controller/metadata.h" +#include "md_parser.h" + +#include "libcamera/internal/v4l2_videodevice.h" + +namespace RPiController { + +/* + * The CamHelper class provides a number of facilities that anyone trying + * to drive a camera will need to know, but which are not provided by the + * standard driver framework. Specifically, it provides: + * + * A "CameraMode" structure to describe extra information about the chosen + * mode of the driver. For example, how it is cropped from the full sensor + * area, how it is scaled, whether pixels are averaged compared to the full + * resolution. + * + * The ability to convert between number of lines of exposure and actual + * exposure time, and to convert between the sensor's gain codes and actual + * gains. + * + * A function to return the number of frames of delay between updating exposure, + * analogue gain and vblanking, and for the changes to take effect. For many + * sensors these take the values 2, 1 and 2 respectively, but sensors that are + * different will need to over-ride the default function provided. + * + * A function to query if the sensor outputs embedded data that can be parsed. + * + * A function to return the sensitivity of a given camera mode. + * + * A parser to parse the embedded data buffers provided by some sensors (for + * example, the imx219 does; the ov5647 doesn't). This allows us to know for + * sure the exposure and gain of the frame we're looking at. CamHelper + * provides functions for converting analogue gains to and from the sensor's + * native gain codes. + * + * Finally, a set of functions that determine how to handle the vagaries of + * different camera modules on start-up or when switching modes. Some + * modules may produce one or more frames that are not yet correctly exposed, + * or where the metadata may be suspect. We have the following functions: + * HideFramesStartup(): Tell the pipeline handler not to return this many + * frames at start-up. This can also be used to hide initial frames + * while the AGC and other algorithms are sorting themselves out. + * HideFramesModeSwitch(): Tell the pipeline handler not to return this + * many frames after a mode switch (other than start-up). Some sensors + * may produce innvalid frames after a mode switch; others may not. + * MistrustFramesStartup(): At start-up a sensor may return frames for + * which we should not run any control algorithms (for example, metadata + * may be invalid). + * MistrustFramesModeSwitch(): The number of frames, after a mode switch + * (other than start-up), for which control algorithms should not run + * (for example, metadata may be unreliable). + */ + +class CamHelper +{ +public: + static CamHelper *create(std::string const &camName); + CamHelper(std::unique_ptr parser, unsigned int frameIntegrationDiff); + virtual ~CamHelper(); + void setCameraMode(const CameraMode &mode); + virtual void prepare(libcamera::Span buffer, + Metadata &metadata); + virtual void process(StatisticsPtr &stats, Metadata &metadata); + virtual uint32_t exposureLines(const libcamera::utils::Duration exposure, + const libcamera::utils::Duration lineLength) const; + virtual libcamera::utils::Duration exposure(uint32_t exposureLines, + const libcamera::utils::Duration lineLength) const; + virtual std::pair getBlanking(libcamera::utils::Duration &exposure, + libcamera::utils::Duration minFrameDuration, + libcamera::utils::Duration maxFrameDuration) const; + libcamera::utils::Duration hblankToLineLength(uint32_t hblank) const; + uint32_t lineLengthToHblank(const libcamera::utils::Duration &duration) const; + libcamera::utils::Duration lineLengthPckToDuration(uint32_t lineLengthPck) const; + virtual uint32_t gainCode(double gain) const = 0; + virtual double gain(uint32_t gainCode) const = 0; + virtual void getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const; + virtual bool sensorEmbeddedDataPresent() const; + virtual double getModeSensitivity(const CameraMode &mode) const; + virtual unsigned int hideFramesStartup() const; + virtual unsigned int hideFramesModeSwitch() const; + virtual unsigned int mistrustFramesStartup() const; + virtual unsigned int mistrustFramesModeSwitch() const; + +protected: + void parseEmbeddedData(libcamera::Span buffer, + Metadata &metadata); + virtual void populateMetadata(const MdParser::RegisterMap ®isters, + Metadata &metadata) const; + + std::unique_ptr parser_; + CameraMode mode_; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + unsigned int frameIntegrationDiff_; +}; + +/* + * This is for registering camera helpers with the system, so that the + * CamHelper::Create function picks them up automatically. + */ + +typedef CamHelper *(*CamHelperCreateFunc)(); +struct RegisterCamHelper +{ + RegisterCamHelper(char const *camName, + CamHelperCreateFunc createFunc); +}; + +} /* namespace RPi */ diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx219.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx219.cpp new file mode 100644 index 00000000..c3337ed0 --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper_imx219.cpp @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi Ltd + * + * cam_helper_imx219.cpp - camera helper for imx219 sensor + */ + +#include +#include +#include +#include + +/* + * We have observed that the imx219 embedded data stream randomly returns junk + * register values. Do not rely on embedded data until this has been resolved. + */ +#define ENABLE_EMBEDDED_DATA 0 + +#include "cam_helper.h" +#if ENABLE_EMBEDDED_DATA +#include "md_parser.h" +#endif + +using namespace RPiController; + +/* + * We care about one gain register and a pair of exposure registers. Their I2C + * addresses from the Sony IMX219 datasheet: + */ +constexpr uint32_t gainReg = 0x157; +constexpr uint32_t expHiReg = 0x15a; +constexpr uint32_t expLoReg = 0x15b; +constexpr uint32_t frameLengthHiReg = 0x160; +constexpr uint32_t frameLengthLoReg = 0x161; +constexpr uint32_t lineLengthHiReg = 0x162; +constexpr uint32_t lineLengthLoReg = 0x163; +constexpr std::initializer_list registerList [[maybe_unused]] + = { expHiReg, expLoReg, gainReg, frameLengthHiReg, frameLengthLoReg, + lineLengthHiReg, lineLengthLoReg }; + +class CamHelperImx219 : public CamHelper +{ +public: + CamHelperImx219(); + uint32_t gainCode(double gain) const override; + double gain(uint32_t gainCode) const override; + unsigned int mistrustFramesModeSwitch() const override; + bool sensorEmbeddedDataPresent() const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 4; + + void populateMetadata(const MdParser::RegisterMap ®isters, + Metadata &metadata) const override; +}; + +CamHelperImx219::CamHelperImx219() +#if ENABLE_EMBEDDED_DATA + : CamHelper(std::make_unique(registerList), frameIntegrationDiff) +#else + : CamHelper({}, frameIntegrationDiff) +#endif +{ +} + +uint32_t CamHelperImx219::gainCode(double gain) const +{ + return (uint32_t)(256 - 256 / gain); +} + +double CamHelperImx219::gain(uint32_t gainCode) const +{ + return 256.0 / (256 - gainCode); +} + +unsigned int CamHelperImx219::mistrustFramesModeSwitch() const +{ + /* + * For reasons unknown, we do occasionally get a bogus metadata frame + * at a mode switch (though not at start-up). Possibly warrants some + * investigation, though not a big deal. + */ + return 1; +} + +bool CamHelperImx219::sensorEmbeddedDataPresent() const +{ + return ENABLE_EMBEDDED_DATA; +} + +void CamHelperImx219::populateMetadata(const MdParser::RegisterMap ®isters, + Metadata &metadata) const +{ + DeviceStatus deviceStatus; + + deviceStatus.lineLength = lineLengthPckToDuration(registers.at(lineLengthHiReg) * 256 + + registers.at(lineLengthLoReg)); + deviceStatus.shutterSpeed = exposure(registers.at(expHiReg) * 256 + registers.at(expLoReg), + deviceStatus.lineLength); + deviceStatus.analogueGain = gain(registers.at(gainReg)); + deviceStatus.frameLength = registers.at(frameLengthHiReg) * 256 + registers.at(frameLengthLoReg); + + metadata.set("device.status", deviceStatus); +} + +static CamHelper *create() +{ + return new CamHelperImx219(); +} + +static RegisterCamHelper reg("imx219", &create); diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx290.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx290.cpp new file mode 100644 index 00000000..7d6f5b54 --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper_imx290.cpp @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2021, Raspberry Pi Ltd + * + * cam_helper_imx290.cpp - camera helper for imx290 sensor + */ + +#include + +#include "cam_helper.h" + +using namespace RPiController; + +class CamHelperImx290 : public CamHelper +{ +public: + CamHelperImx290(); + uint32_t gainCode(double gain) const override; + double gain(uint32_t gainCode) const override; + void getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const override; + unsigned int hideFramesModeSwitch() const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 2; +}; + +CamHelperImx290::CamHelperImx290() + : CamHelper({}, frameIntegrationDiff) +{ +} + +uint32_t CamHelperImx290::gainCode(double gain) const +{ + int code = 66.6667 * log10(gain); + return std::max(0, std::min(code, 0xf0)); +} + +double CamHelperImx290::gain(uint32_t gainCode) const +{ + return pow(10, 0.015 * gainCode); +} + +void CamHelperImx290::getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const +{ + exposureDelay = 2; + gainDelay = 2; + vblankDelay = 2; + hblankDelay = 2; +} + +unsigned int CamHelperImx290::hideFramesModeSwitch() const +{ + /* After a mode switch, we seem to get 1 bad frame. */ + return 1; +} + +static CamHelper *create() +{ + return new CamHelperImx290(); +} + +static RegisterCamHelper reg("imx290", &create); diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx296.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx296.cpp new file mode 100644 index 00000000..ecb845e7 --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper_imx296.cpp @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2020, Raspberry Pi Ltd + * + * cam_helper_imx296.cpp - Camera helper for IMX296 sensor + */ + +#include +#include +#include + +#include "cam_helper.h" + +using namespace RPiController; +using libcamera::utils::Duration; +using namespace std::literals::chrono_literals; + +class CamHelperImx296 : public CamHelper +{ +public: + CamHelperImx296(); + uint32_t gainCode(double gain) const override; + double gain(uint32_t gainCode) const override; + uint32_t exposureLines(const Duration exposure, const Duration lineLength) const override; + Duration exposure(uint32_t exposureLines, const Duration lineLength) const override; + void getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const override; + +private: + static constexpr uint32_t minExposureLines = 1; + static constexpr uint32_t maxGainCode = 239; + static constexpr Duration timePerLine = 550.0 / 37.125e6 * 1.0s; + + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 4; +}; + +CamHelperImx296::CamHelperImx296() + : CamHelper(nullptr, frameIntegrationDiff) +{ +} + +uint32_t CamHelperImx296::gainCode(double gain) const +{ + uint32_t code = 20 * std::log10(gain) * 10; + return std::min(code, maxGainCode); +} + +double CamHelperImx296::gain(uint32_t gainCode) const +{ + return std::pow(10.0, gainCode / 200.0); +} + +uint32_t CamHelperImx296::exposureLines(const Duration exposure, + [[maybe_unused]] const Duration lineLength) const +{ + return std::max(minExposureLines, (exposure - 14.26us) / timePerLine); +} + +Duration CamHelperImx296::exposure(uint32_t exposureLines, + [[maybe_unused]] const Duration lineLength) const +{ + return std::max(minExposureLines, exposureLines) * timePerLine + 14.26us; +} + +void CamHelperImx296::getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const +{ + exposureDelay = 2; + gainDelay = 2; + vblankDelay = 2; + hblankDelay = 2; +} + +static CamHelper *create() +{ + return new CamHelperImx296(); +} + +static RegisterCamHelper reg("imx296", &create); diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx477.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx477.cpp new file mode 100644 index 00000000..bc769ca7 --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper_imx477.cpp @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2020, Raspberry Pi Ltd + * + * cam_helper_imx477.cpp - camera helper for imx477 sensor + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "cam_helper.h" +#include "md_parser.h" + +using namespace RPiController; +using namespace libcamera; +using libcamera::utils::Duration; + +namespace libcamera { +LOG_DECLARE_CATEGORY(IPARPI) +} + +/* + * We care about two gain registers and a pair of exposure registers. Their + * I2C addresses from the Sony IMX477 datasheet: + */ +constexpr uint32_t expHiReg = 0x0202; +constexpr uint32_t expLoReg = 0x0203; +constexpr uint32_t gainHiReg = 0x0204; +constexpr uint32_t gainLoReg = 0x0205; +constexpr uint32_t frameLengthHiReg = 0x0340; +constexpr uint32_t frameLengthLoReg = 0x0341; +constexpr uint32_t lineLengthHiReg = 0x0342; +constexpr uint32_t lineLengthLoReg = 0x0343; +constexpr uint32_t temperatureReg = 0x013a; +constexpr std::initializer_list registerList = + { expHiReg, expLoReg, gainHiReg, gainLoReg, frameLengthHiReg, frameLengthLoReg, + lineLengthHiReg, lineLengthLoReg, temperatureReg }; + +class CamHelperImx477 : public CamHelper +{ +public: + CamHelperImx477(); + uint32_t gainCode(double gain) const override; + double gain(uint32_t gainCode) const override; + void prepare(libcamera::Span buffer, Metadata &metadata) override; + std::pair getBlanking(Duration &exposure, Duration minFrameDuration, + Duration maxFrameDuration) const override; + void getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const override; + bool sensorEmbeddedDataPresent() const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 22; + /* Maximum frame length allowable for long exposure calculations. */ + static constexpr int frameLengthMax = 0xffdc; + /* Largest long exposure scale factor given as a left shift on the frame length. */ + static constexpr int longExposureShiftMax = 7; + + void populateMetadata(const MdParser::RegisterMap ®isters, + Metadata &metadata) const override; +}; + +CamHelperImx477::CamHelperImx477() + : CamHelper(std::make_unique(registerList), frameIntegrationDiff) +{ +} + +uint32_t CamHelperImx477::gainCode(double gain) const +{ + return static_cast(1024 - 1024 / gain); +} + +double CamHelperImx477::gain(uint32_t gainCode) const +{ + return 1024.0 / (1024 - gainCode); +} + +void CamHelperImx477::prepare(libcamera::Span buffer, Metadata &metadata) +{ + MdParser::RegisterMap registers; + DeviceStatus deviceStatus; + + if (metadata.get("device.status", deviceStatus)) { + LOG(IPARPI, Error) << "DeviceStatus not found from DelayedControls"; + return; + } + + parseEmbeddedData(buffer, metadata); + + /* + * The DeviceStatus struct is first populated with values obtained from + * DelayedControls. If this reports frame length is > frameLengthMax, + * it means we are using a long exposure mode. Since the long exposure + * scale factor is not returned back through embedded data, we must rely + * on the existing exposure lines and frame length values returned by + * DelayedControls. + * + * Otherwise, all values are updated with what is reported in the + * embedded data. + */ + if (deviceStatus.frameLength > frameLengthMax) { + DeviceStatus parsedDeviceStatus; + + metadata.get("device.status", parsedDeviceStatus); + parsedDeviceStatus.shutterSpeed = deviceStatus.shutterSpeed; + parsedDeviceStatus.frameLength = deviceStatus.frameLength; + metadata.set("device.status", parsedDeviceStatus); + + LOG(IPARPI, Debug) << "Metadata updated for long exposure: " + << parsedDeviceStatus; + } +} + +std::pair CamHelperImx477::getBlanking(Duration &exposure, + Duration minFrameDuration, + Duration maxFrameDuration) const +{ + uint32_t frameLength, exposureLines; + unsigned int shift = 0; + + auto [vblank, hblank] = CamHelper::getBlanking(exposure, minFrameDuration, + maxFrameDuration); + + frameLength = mode_.height + vblank; + Duration lineLength = hblankToLineLength(hblank); + + /* + * Check if the frame length calculated needs to be setup for long + * exposure mode. This will require us to use a long exposure scale + * factor provided by a shift operation in the sensor. + */ + while (frameLength > frameLengthMax) { + if (++shift > longExposureShiftMax) { + shift = longExposureShiftMax; + frameLength = frameLengthMax; + break; + } + frameLength >>= 1; + } + + if (shift) { + /* Account for any rounding in the scaled frame length value. */ + frameLength <<= shift; + exposureLines = CamHelperImx477::exposureLines(exposure, lineLength); + exposureLines = std::min(exposureLines, frameLength - frameIntegrationDiff); + exposure = CamHelperImx477::exposure(exposureLines, lineLength); + } + + return { frameLength - mode_.height, hblank }; +} + +void CamHelperImx477::getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const +{ + exposureDelay = 2; + gainDelay = 2; + vblankDelay = 3; + hblankDelay = 3; +} + +bool CamHelperImx477::sensorEmbeddedDataPresent() const +{ + return true; +} + +void CamHelperImx477::populateMetadata(const MdParser::RegisterMap ®isters, + Metadata &metadata) const +{ + DeviceStatus deviceStatus; + + deviceStatus.lineLength = lineLengthPckToDuration(registers.at(lineLengthHiReg) * 256 + + registers.at(lineLengthLoReg)); + deviceStatus.shutterSpeed = exposure(registers.at(expHiReg) * 256 + registers.at(expLoReg), + deviceStatus.lineLength); + deviceStatus.analogueGain = gain(registers.at(gainHiReg) * 256 + registers.at(gainLoReg)); + deviceStatus.frameLength = registers.at(frameLengthHiReg) * 256 + registers.at(frameLengthLoReg); + deviceStatus.sensorTemperature = std::clamp(registers.at(temperatureReg), -20, 80); + + metadata.set("device.status", deviceStatus); +} + +static CamHelper *create() +{ + return new CamHelperImx477(); +} + +static RegisterCamHelper reg("imx477", &create); diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx519.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx519.cpp new file mode 100644 index 00000000..c7262aa0 --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper_imx519.cpp @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Based on cam_helper_imx477.cpp + * Copyright (C) 2020, Raspberry Pi Ltd + * + * cam_helper_imx519.cpp - camera helper for imx519 sensor + * Copyright (C) 2021, Arducam Technology co., Ltd. + */ + +#include +#include +#include +#include +#include + +#include + +#include "cam_helper.h" +#include "md_parser.h" + +using namespace RPiController; +using namespace libcamera; +using libcamera::utils::Duration; + +namespace libcamera { +LOG_DECLARE_CATEGORY(IPARPI) +} + +/* + * We care about two gain registers and a pair of exposure registers. Their + * I2C addresses from the Sony IMX519 datasheet: + */ +constexpr uint32_t expHiReg = 0x0202; +constexpr uint32_t expLoReg = 0x0203; +constexpr uint32_t gainHiReg = 0x0204; +constexpr uint32_t gainLoReg = 0x0205; +constexpr uint32_t frameLengthHiReg = 0x0340; +constexpr uint32_t frameLengthLoReg = 0x0341; +constexpr uint32_t lineLengthHiReg = 0x0342; +constexpr uint32_t lineLengthLoReg = 0x0343; +constexpr std::initializer_list registerList = + { expHiReg, expLoReg, gainHiReg, gainLoReg, frameLengthHiReg, frameLengthLoReg, + lineLengthHiReg, lineLengthLoReg }; + +class CamHelperImx519 : public CamHelper +{ +public: + CamHelperImx519(); + uint32_t gainCode(double gain) const override; + double gain(uint32_t gainCode) const override; + void prepare(libcamera::Span buffer, Metadata &metadata) override; + std::pair getBlanking(Duration &exposure, Duration minFrameDuration, + Duration maxFrameDuration) const override; + void getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const override; + bool sensorEmbeddedDataPresent() const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 32; + /* Maximum frame length allowable for long exposure calculations. */ + static constexpr int frameLengthMax = 0xffdc; + /* Largest long exposure scale factor given as a left shift on the frame length. */ + static constexpr int longExposureShiftMax = 7; + + void populateMetadata(const MdParser::RegisterMap ®isters, + Metadata &metadata) const override; +}; + +CamHelperImx519::CamHelperImx519() + : CamHelper(std::make_unique(registerList), frameIntegrationDiff) +{ +} + +uint32_t CamHelperImx519::gainCode(double gain) const +{ + return static_cast(1024 - 1024 / gain); +} + +double CamHelperImx519::gain(uint32_t gainCode) const +{ + return 1024.0 / (1024 - gainCode); +} + +void CamHelperImx519::prepare(libcamera::Span buffer, Metadata &metadata) +{ + MdParser::RegisterMap registers; + DeviceStatus deviceStatus; + + if (metadata.get("device.status", deviceStatus)) { + LOG(IPARPI, Error) << "DeviceStatus not found from DelayedControls"; + return; + } + + parseEmbeddedData(buffer, metadata); + + /* + * The DeviceStatus struct is first populated with values obtained from + * DelayedControls. If this reports frame length is > frameLengthMax, + * it means we are using a long exposure mode. Since the long exposure + * scale factor is not returned back through embedded data, we must rely + * on the existing exposure lines and frame length values returned by + * DelayedControls. + * + * Otherwise, all values are updated with what is reported in the + * embedded data. + */ + if (deviceStatus.frameLength > frameLengthMax) { + DeviceStatus parsedDeviceStatus; + + metadata.get("device.status", parsedDeviceStatus); + parsedDeviceStatus.shutterSpeed = deviceStatus.shutterSpeed; + parsedDeviceStatus.frameLength = deviceStatus.frameLength; + metadata.set("device.status", parsedDeviceStatus); + + LOG(IPARPI, Debug) << "Metadata updated for long exposure: " + << parsedDeviceStatus; + } +} + +std::pair CamHelperImx519::getBlanking(Duration &exposure, + Duration minFrameDuration, + Duration maxFrameDuration) const +{ + uint32_t frameLength, exposureLines; + unsigned int shift = 0; + + auto [vblank, hblank] = CamHelper::getBlanking(exposure, minFrameDuration, + maxFrameDuration); + + frameLength = mode_.height + vblank; + Duration lineLength = hblankToLineLength(hblank); + + /* + * Check if the frame length calculated needs to be setup for long + * exposure mode. This will require us to use a long exposure scale + * factor provided by a shift operation in the sensor. + */ + while (frameLength > frameLengthMax) { + if (++shift > longExposureShiftMax) { + shift = longExposureShiftMax; + frameLength = frameLengthMax; + break; + } + frameLength >>= 1; + } + + if (shift) { + /* Account for any rounding in the scaled frame length value. */ + frameLength <<= shift; + exposureLines = CamHelperImx519::exposureLines(exposure, lineLength); + exposureLines = std::min(exposureLines, frameLength - frameIntegrationDiff); + exposure = CamHelperImx519::exposure(exposureLines, lineLength); + } + + return { frameLength - mode_.height, hblank }; +} + +void CamHelperImx519::getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const +{ + exposureDelay = 2; + gainDelay = 2; + vblankDelay = 3; + hblankDelay = 3; +} + +bool CamHelperImx519::sensorEmbeddedDataPresent() const +{ + return true; +} + +void CamHelperImx519::populateMetadata(const MdParser::RegisterMap ®isters, + Metadata &metadata) const +{ + DeviceStatus deviceStatus; + + deviceStatus.lineLength = lineLengthPckToDuration(registers.at(lineLengthHiReg) * 256 + + registers.at(lineLengthLoReg)); + deviceStatus.shutterSpeed = exposure(registers.at(expHiReg) * 256 + registers.at(expLoReg), + deviceStatus.lineLength); + deviceStatus.analogueGain = gain(registers.at(gainHiReg) * 256 + registers.at(gainLoReg)); + deviceStatus.frameLength = registers.at(frameLengthHiReg) * 256 + registers.at(frameLengthLoReg); + + metadata.set("device.status", deviceStatus); +} + +static CamHelper *create() +{ + return new CamHelperImx519(); +} + +static RegisterCamHelper reg("imx519", &create); diff --git a/src/ipa/rpi/cam_helper/cam_helper_imx708.cpp b/src/ipa/rpi/cam_helper/cam_helper_imx708.cpp new file mode 100644 index 00000000..641ba18f --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper_imx708.cpp @@ -0,0 +1,359 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2022, Raspberry Pi Ltd + * + * cam_helper_imx708.cpp - camera helper for imx708 sensor + */ + +#include +#include +#include +#include + +#include + +#include "controller/pdaf_data.h" + +#include "cam_helper.h" +#include "md_parser.h" + +using namespace RPiController; +using namespace libcamera; +using libcamera::utils::Duration; + +namespace libcamera { +LOG_DECLARE_CATEGORY(IPARPI) +} + +/* + * We care about two gain registers and a pair of exposure registers. Their + * I2C addresses from the Sony imx708 datasheet: + */ +constexpr uint32_t expHiReg = 0x0202; +constexpr uint32_t expLoReg = 0x0203; +constexpr uint32_t gainHiReg = 0x0204; +constexpr uint32_t gainLoReg = 0x0205; +constexpr uint32_t frameLengthHiReg = 0x0340; +constexpr uint32_t frameLengthLoReg = 0x0341; +constexpr uint32_t lineLengthHiReg = 0x0342; +constexpr uint32_t lineLengthLoReg = 0x0343; +constexpr uint32_t temperatureReg = 0x013a; +constexpr std::initializer_list registerList = + { expHiReg, expLoReg, gainHiReg, gainLoReg, lineLengthHiReg, + lineLengthLoReg, frameLengthHiReg, frameLengthLoReg, temperatureReg }; + +class CamHelperImx708 : public CamHelper +{ +public: + CamHelperImx708(); + uint32_t gainCode(double gain) const override; + double gain(uint32_t gain_code) const override; + void prepare(libcamera::Span buffer, Metadata &metadata) override; + void process(StatisticsPtr &stats, Metadata &metadata) override; + std::pair getBlanking(Duration &exposure, Duration minFrameDuration, + Duration maxFrameDuration) const override; + void getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const override; + bool sensorEmbeddedDataPresent() const override; + double getModeSensitivity(const CameraMode &mode) const override; + unsigned int hideFramesModeSwitch() const override { return 1; } // seems to be required for HDR + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 22; + /* Maximum frame length allowable for long exposure calculations. */ + static constexpr int frameLengthMax = 0xffdc; + /* Largest long exposure scale factor given as a left shift on the frame length. */ + static constexpr int longExposureShiftMax = 7; + + static constexpr int pdafStatsRows = 12; + static constexpr int pdafStatsCols = 16; + + void populateMetadata(const MdParser::RegisterMap ®isters, + Metadata &metadata) const override; + + static bool parsePdafData(const uint8_t *ptr, size_t len, unsigned bpp, + PdafRegions &pdaf); + + bool parseAEHist(const uint8_t *ptr, size_t len, unsigned bpp); + void putAGCStatistics(StatisticsPtr stats); + + Histogram aeHistLinear_; + uint32_t aeHistAverage_; + bool aeHistValid_; +}; + +CamHelperImx708::CamHelperImx708() + : CamHelper(std::make_unique(registerList), frameIntegrationDiff), + aeHistLinear_{}, aeHistAverage_(0), aeHistValid_(false) +{ +} + +uint32_t CamHelperImx708::gainCode(double gain) const +{ + return static_cast(1024 - 1024 / gain); +} + +double CamHelperImx708::gain(uint32_t gain_code) const +{ + return 1024.0 / (1024 - gain_code); +} + +void CamHelperImx708::prepare(libcamera::Span buffer, Metadata &metadata) +{ + MdParser::RegisterMap registers; + DeviceStatus deviceStatus; + + LOG(IPARPI, Debug) << "Embedded buffer size: " << buffer.size(); + + if (metadata.get("device.status", deviceStatus)) { + LOG(IPARPI, Error) << "DeviceStatus not found from DelayedControls"; + return; + } + + parseEmbeddedData(buffer, metadata); + + /* + * Parse PDAF data, which we expect to occupy the third scanline + * of embedded data. As PDAF is quite sensor-specific, it's parsed here. + */ + size_t bytesPerLine = (mode_.width * mode_.bitdepth) >> 3; + + if (buffer.size() > 2 * bytesPerLine) { + PdafRegions pdaf; + if (parsePdafData(&buffer[2 * bytesPerLine], + buffer.size() - 2 * bytesPerLine, + mode_.bitdepth, pdaf)) + metadata.set("pdaf.regions", pdaf); + } + + /* Parse AE-HIST data where present */ + if (buffer.size() > 3 * bytesPerLine) { + aeHistValid_ = parseAEHist(&buffer[3 * bytesPerLine], + buffer.size() - 3 * bytesPerLine, + mode_.bitdepth); + } + + /* + * The DeviceStatus struct is first populated with values obtained from + * DelayedControls. If this reports frame length is > frameLengthMax, + * it means we are using a long exposure mode. Since the long exposure + * scale factor is not returned back through embedded data, we must rely + * on the existing exposure lines and frame length values returned by + * DelayedControls. + * + * Otherwise, all values are updated with what is reported in the + * embedded data. + */ + if (deviceStatus.frameLength > frameLengthMax) { + DeviceStatus parsedDeviceStatus; + + metadata.get("device.status", parsedDeviceStatus); + parsedDeviceStatus.shutterSpeed = deviceStatus.shutterSpeed; + parsedDeviceStatus.frameLength = deviceStatus.frameLength; + metadata.set("device.status", parsedDeviceStatus); + + LOG(IPARPI, Debug) << "Metadata updated for long exposure: " + << parsedDeviceStatus; + } +} + +void CamHelperImx708::process(StatisticsPtr &stats, [[maybe_unused]] Metadata &metadata) +{ + if (aeHistValid_) + putAGCStatistics(stats); +} + +std::pair CamHelperImx708::getBlanking(Duration &exposure, + Duration minFrameDuration, + Duration maxFrameDuration) const +{ + uint32_t frameLength, exposureLines; + unsigned int shift = 0; + + auto [vblank, hblank] = CamHelper::getBlanking(exposure, minFrameDuration, + maxFrameDuration); + + frameLength = mode_.height + vblank; + Duration lineLength = hblankToLineLength(hblank); + + /* + * Check if the frame length calculated needs to be setup for long + * exposure mode. This will require us to use a long exposure scale + * factor provided by a shift operation in the sensor. + */ + while (frameLength > frameLengthMax) { + if (++shift > longExposureShiftMax) { + shift = longExposureShiftMax; + frameLength = frameLengthMax; + break; + } + frameLength >>= 1; + } + + if (shift) { + /* Account for any rounding in the scaled frame length value. */ + frameLength <<= shift; + exposureLines = CamHelper::exposureLines(exposure, lineLength); + exposureLines = std::min(exposureLines, frameLength - frameIntegrationDiff); + exposure = CamHelper::exposure(exposureLines, lineLength); + } + + return { frameLength - mode_.height, hblank }; +} + +void CamHelperImx708::getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const +{ + exposureDelay = 2; + gainDelay = 2; + vblankDelay = 3; + hblankDelay = 3; +} + +bool CamHelperImx708::sensorEmbeddedDataPresent() const +{ + return true; +} + +double CamHelperImx708::getModeSensitivity(const CameraMode &mode) const +{ + /* In binned modes, sensitivity increases by a factor of 2 */ + return (mode.width > 2304) ? 1.0 : 2.0; +} + +void CamHelperImx708::populateMetadata(const MdParser::RegisterMap ®isters, + Metadata &metadata) const +{ + DeviceStatus deviceStatus; + + deviceStatus.lineLength = lineLengthPckToDuration(registers.at(lineLengthHiReg) * 256 + + registers.at(lineLengthLoReg)); + deviceStatus.shutterSpeed = exposure(registers.at(expHiReg) * 256 + registers.at(expLoReg), + deviceStatus.lineLength); + deviceStatus.analogueGain = gain(registers.at(gainHiReg) * 256 + registers.at(gainLoReg)); + deviceStatus.frameLength = registers.at(frameLengthHiReg) * 256 + registers.at(frameLengthLoReg); + deviceStatus.sensorTemperature = std::clamp(registers.at(temperatureReg), -20, 80); + + metadata.set("device.status", deviceStatus); +} + +bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len, + unsigned bpp, PdafRegions &pdaf) +{ + size_t step = bpp >> 1; /* bytes per PDAF grid entry */ + + if (bpp < 10 || bpp > 12 || len < 194 * step || ptr[0] != 0 || ptr[1] >= 0x40) { + LOG(IPARPI, Error) << "PDAF data in unsupported format"; + return false; + } + + pdaf.init({ pdafStatsCols, pdafStatsRows }); + + ptr += 2 * step; + for (unsigned i = 0; i < pdafStatsRows; ++i) { + for (unsigned j = 0; j < pdafStatsCols; ++j) { + unsigned c = (ptr[0] << 3) | (ptr[1] >> 5); + int p = (((ptr[1] & 0x0F) - (ptr[1] & 0x10)) << 6) | (ptr[2] >> 2); + PdafData pdafData; + pdafData.conf = c; + pdafData.phase = c ? p : 0; + pdaf.set(libcamera::Point(j, i), { pdafData, 1, 0 }); + ptr += step; + } + } + + return true; +} + +bool CamHelperImx708::parseAEHist(const uint8_t *ptr, size_t len, unsigned bpp) +{ + static constexpr unsigned int PipelineBits = Statistics::NormalisationFactorPow2; + + uint64_t count = 0, sum = 0; + size_t step = bpp >> 1; /* bytes per histogram bin */ + uint32_t hist[128]; + + if (len < 144 * step) + return false; + + /* + * Read the 128 bin linear histogram, which by default covers + * the full range of the HDR shortest exposure (small values are + * expected to dominate, so pixel-value resolution will be poor). + */ + for (unsigned i = 0; i < 128; ++i) { + if (ptr[3] != 0x55) + return false; + uint32_t c = (ptr[0] << 14) + (ptr[1] << 6) + (ptr[2] >> 2); + hist[i] = c >> 2; /* pixels to quads */ + if (i != 0) { + count += c; + sum += c * + (i * (1u << (PipelineBits - 7)) + + (1u << (PipelineBits - 8))); + } + ptr += step; + } + + /* + * Now use the first 9 bins of the log histogram (these should be + * subdivisions of the smallest linear bin), to get a more accurate + * average value. Don't assume that AEHIST1_AVERAGE is present. + */ + for (unsigned i = 0; i < 9; ++i) { + if (ptr[3] != 0x55) + return false; + uint32_t c = (ptr[0] << 14) + (ptr[1] << 6) + (ptr[2] >> 2); + count += c; + sum += c * + ((3u << PipelineBits) >> (17 - i)); + ptr += step; + } + if ((unsigned)((ptr[0] << 12) + (ptr[1] << 4) + (ptr[2] >> 4)) != + hist[1]) { + LOG(IPARPI, Error) << "Lin/Log histogram mismatch"; + return false; + } + + aeHistLinear_ = Histogram(hist, 128); + aeHistAverage_ = count ? (sum / count) : 0; + + return count != 0; +} + +void CamHelperImx708::putAGCStatistics(StatisticsPtr stats) +{ + /* + * For HDR mode, copy sensor's AE/AGC statistics over ISP's, so the + * AGC algorithm sees a linear response to exposure and gain changes. + * + * Histogram: Just copy the "raw" histogram over the tone-mapped one, + * although they have different distributions (raw values are lower). + * Tuning should either ignore it, or constrain for highlights only. + * + * Average: Overwrite all regional averages with a global raw average, + * scaled by a fiddle-factor so that a conventional (non-HDR) y_target + * of e.g. 0.17 will map to a suitable level for HDR. + */ + stats->yHist = aeHistLinear_; + + constexpr unsigned int HdrHeadroomFactor = 4; + uint64_t v = HdrHeadroomFactor * aeHistAverage_; + for (auto ®ion : stats->agcRegions) { + region.val.rSum = region.val.gSum = region.val.bSum = region.counted * v; + } +} + +static CamHelper *create() +{ + return new CamHelperImx708(); +} + +static RegisterCamHelper reg("imx708", &create); +static RegisterCamHelper regWide("imx708_wide", &create); +static RegisterCamHelper regNoIr("imx708_noir", &create); +static RegisterCamHelper regWideNoIr("imx708_wide_noir", &create); diff --git a/src/ipa/rpi/cam_helper/cam_helper_ov5647.cpp b/src/ipa/rpi/cam_helper/cam_helper_ov5647.cpp new file mode 100644 index 00000000..5a99083d --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper_ov5647.cpp @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi Ltd + * + * cam_helper_ov5647.cpp - camera information for ov5647 sensor + */ + +#include + +#include "cam_helper.h" + +using namespace RPiController; + +class CamHelperOv5647 : public CamHelper +{ +public: + CamHelperOv5647(); + uint32_t gainCode(double gain) const override; + double gain(uint32_t gainCode) const override; + void getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const override; + unsigned int hideFramesStartup() const override; + unsigned int hideFramesModeSwitch() const override; + unsigned int mistrustFramesStartup() const override; + unsigned int mistrustFramesModeSwitch() const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 4; +}; + +/* + * OV5647 doesn't output metadata, so we have to use the "unicam parser" which + * works by counting frames. + */ + +CamHelperOv5647::CamHelperOv5647() + : CamHelper({}, frameIntegrationDiff) +{ +} + +uint32_t CamHelperOv5647::gainCode(double gain) const +{ + return static_cast(gain * 16.0); +} + +double CamHelperOv5647::gain(uint32_t gainCode) const +{ + return static_cast(gainCode) / 16.0; +} + +void CamHelperOv5647::getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const +{ + /* + * We run this sensor in a mode where the gain delay is bumped up to + * 2. It seems to be the only way to make the delays "predictable". + */ + exposureDelay = 2; + gainDelay = 2; + vblankDelay = 2; + hblankDelay = 2; +} + +unsigned int CamHelperOv5647::hideFramesStartup() const +{ + /* + * On startup, we get a couple of under-exposed frames which + * we don't want shown. + */ + return 2; +} + +unsigned int CamHelperOv5647::hideFramesModeSwitch() const +{ + /* + * After a mode switch, we get a couple of under-exposed frames which + * we don't want shown. + */ + return 2; +} + +unsigned int CamHelperOv5647::mistrustFramesStartup() const +{ + /* + * First couple of frames are under-exposed and are no good for control + * algos. + */ + return 2; +} + +unsigned int CamHelperOv5647::mistrustFramesModeSwitch() const +{ + /* + * First couple of frames are under-exposed even after a simple + * mode switch, and are no good for control algos. + */ + return 2; +} + +static CamHelper *create() +{ + return new CamHelperOv5647(); +} + +static RegisterCamHelper reg("ov5647", &create); diff --git a/src/ipa/rpi/cam_helper/cam_helper_ov9281.cpp b/src/ipa/rpi/cam_helper/cam_helper_ov9281.cpp new file mode 100644 index 00000000..86c5bc4c --- /dev/null +++ b/src/ipa/rpi/cam_helper/cam_helper_ov9281.cpp @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2021, Raspberry Pi Ltd + * + * cam_helper_ov9281.cpp - camera information for ov9281 sensor + */ + +#include + +#include "cam_helper.h" + +using namespace RPiController; + +class CamHelperOv9281 : public CamHelper +{ +public: + CamHelperOv9281(); + uint32_t gainCode(double gain) const override; + double gain(uint32_t gainCode) const override; + void getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const override; + +private: + /* + * Smallest difference between the frame length and integration time, + * in units of lines. + */ + static constexpr int frameIntegrationDiff = 4; +}; + +/* + * OV9281 doesn't output metadata, so we have to use the "unicam parser" which + * works by counting frames. + */ + +CamHelperOv9281::CamHelperOv9281() + : CamHelper({}, frameIntegrationDiff) +{ +} + +uint32_t CamHelperOv9281::gainCode(double gain) const +{ + return static_cast(gain * 16.0); +} + +double CamHelperOv9281::gain(uint32_t gainCode) const +{ + return static_cast(gainCode) / 16.0; +} + +void CamHelperOv9281::getDelays(int &exposureDelay, int &gainDelay, + int &vblankDelay, int &hblankDelay) const +{ + /* The driver appears to behave as follows: */ + exposureDelay = 2; + gainDelay = 2; + vblankDelay = 2; + hblankDelay = 2; +} + +static CamHelper *create() +{ + return new CamHelperOv9281(); +} + +static RegisterCamHelper reg("ov9281", &create); diff --git a/src/ipa/rpi/cam_helper/md_parser.h b/src/ipa/rpi/cam_helper/md_parser.h new file mode 100644 index 00000000..77d557aa --- /dev/null +++ b/src/ipa/rpi/cam_helper/md_parser.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019, Raspberry Pi Ltd + * + * md_parser.h - image sensor metadata parser interface + */ +#pragma once + +#include +#include +#include +#include + +#include + +/* + * Camera metadata parser class. Usage as shown below. + * + * Setup: + * + * Usually the metadata parser will be made as part of the CamHelper class so + * application code doesn't have to worry which kind to instantiate. But for + * the sake of example let's suppose we're parsing imx219 metadata. + * + * MdParser *parser = new MdParserSmia({ expHiReg, expLoReg, gainReg }); + * parser->SetBitsPerPixel(bpp); + * parser->SetLineLengthBytes(pitch); + * parser->SetNumLines(2); + * + * Note 1: if you don't know how many lines there are, the size of the input + * buffer is used as a limit instead. + * + * Note 2: if you don't know the line length, you can leave the line length unset + * (or set to zero) and the parser will hunt for the line start instead. + * + * Then on every frame: + * + * RegisterMap registers; + * if (parser->Parse(buffer, registers) != MdParser::OK) + * much badness; + * Metadata metadata; + * CamHelper::PopulateMetadata(registers, metadata); + * + * (Note that the CamHelper class converts to/from exposure lines and time, + * and gain_code / actual gain.) + * + * If you suspect your embedded data may have changed its layout, change any line + * lengths, number of lines, bits per pixel etc. that are different, and + * then: + * + * parser->Reset(); + * + * before calling Parse again. + */ + +namespace RPiController { + +/* Abstract base class from which other metadata parsers are derived. */ + +class MdParser +{ +public: + using RegisterMap = std::map; + + /* + * Parser status codes: + * OK - success + * NOTFOUND - value such as exposure or gain was not found + * ERROR - all other errors + */ + enum Status { + OK = 0, + NOTFOUND = 1, + ERROR = 2 + }; + + MdParser() + : reset_(true), bitsPerPixel_(0), numLines_(0), lineLengthBytes_(0) + { + } + + virtual ~MdParser() = default; + + void reset() + { + reset_ = true; + } + + void setBitsPerPixel(int bpp) + { + bitsPerPixel_ = bpp; + } + + void setNumLines(unsigned int numLines) + { + numLines_ = numLines; + } + + void setLineLengthBytes(unsigned int numBytes) + { + lineLengthBytes_ = numBytes; + } + + virtual Status parse(libcamera::Span buffer, + RegisterMap ®isters) = 0; + +protected: + bool reset_; + int bitsPerPixel_; + unsigned int numLines_; + unsigned int lineLengthBytes_; +}; + +/* + * This isn't a full implementation of a metadata parser for SMIA sensors, + * however, it does provide the findRegs function which will prove useful and + * make it easier to implement parsers for other SMIA-like sensors (see + * md_parser_imx219.cpp for an example). + */ + +class MdParserSmia final : public MdParser +{ +public: + MdParserSmia(std::initializer_list registerList); + + MdParser::Status parse(libcamera::Span buffer, + RegisterMap ®isters) override; + +private: + /* Maps register address to offset in the buffer. */ + using OffsetMap = std::map>; + + /* + * Note that error codes > 0 are regarded as non-fatal; codes < 0 + * indicate a bad data buffer. Status codes are: + * ParseOk - found all registers, much happiness + * MissingRegs - some registers found; should this be a hard error? + * The remaining codes are all hard errors. + */ + enum ParseStatus { + ParseOk = 0, + MissingRegs = 1, + NoLineStart = -1, + IllegalTag = -2, + BadDummy = -3, + BadLineEnd = -4, + BadPadding = -5 + }; + + ParseStatus findRegs(libcamera::Span buffer); + + OffsetMap offsets_; +}; + +} /* namespace RPi */ diff --git a/src/ipa/rpi/cam_helper/md_parser_smia.cpp b/src/ipa/rpi/cam_helper/md_parser_smia.cpp new file mode 100644 index 00000000..210787ed --- /dev/null +++ b/src/ipa/rpi/cam_helper/md_parser_smia.cpp @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: BSD-2-Clause */ +/* + * Copyright (C) 2019-2021, Raspberry Pi Ltd + * + * md_parser_smia.cpp - SMIA specification based embedded data parser + */ + +#include +#include "md_parser.h" + +using namespace RPiController; +using namespace libcamera; + +/* + * This function goes through the embedded data to find the offsets (not + * values!), in the data block, where the values of the given registers can + * subsequently be found. + * + * Embedded data tag bytes, from Sony IMX219 datasheet but general to all SMIA + * sensors, I think. + */ + +constexpr unsigned int LineStart = 0x0a; +constexpr unsigned int LineEndTag = 0x07; +constexpr unsigned int RegHiBits = 0xaa; +constexpr unsigned int RegLowBits = 0xa5; +constexpr unsigned int RegValue = 0x5a; +constexpr unsigned int RegSkip = 0x55; + +MdParserSmia::MdParserSmia(std::initializer_list registerList) +{ + for (auto r : registerList) + offsets_[r] = {}; +} + +MdParser::Status MdParserSmia::parse(libcamera::Span buffer, + RegisterMap ®isters) +{ + if (reset_) { + /* + * Search again through the metadata for all the registers + * requested. + */ + ASSERT(bitsPerPixel_); + + for (const auto &kv : offsets_) + offsets_[kv.first] = {}; + + ParseStatus ret = findRegs(buffer); + /* + * > 0 means "worked partially but parse again next time", + * < 0 means "hard error". + * + * In either case, we retry parsing on the next frame. + */ + if (ret != ParseOk) + return ERROR; + + reset_ = false; + } + + /* Populate the register values requested. */ + registers.clear(); + for (const auto &[reg, offset] : offsets_) { + if (!offset) { + reset_ = true; + return NOTFOUND; + } + registers[reg] = buffer[offset.value()]; + } + + return OK; +} + +MdParserSmia::ParseStatus MdParserSmia::findRegs(libcamera::Span buffer) +{ + ASSERT(offsets_.size()); + + if (buffer[0] != LineStart) + return NoLineStart; + + unsigned int currentOffset = 1; /* after the LineStart */ + unsigned int currentLineStart = 0, currentLine = 0; + unsigned int regNum = 0, regsDone = 0; + + while (1) { + int tag = buffer[currentOffset++]; + + if ((bitsPerPixel_ == 10 && + (currentOffset + 1 - currentLineStart) % 5 == 0) || + (bitsPerPixel_ == 12 && + (currentOffset + 1 - currentLineStart) % 3 == 0)) { + if (buffer[currentOffset++] != RegSkip) + return BadDummy; + } + + int dataByte = buffer[currentOffset++]; + + if (tag == LineEndTag) { + if (dataByte != LineEndTag) + return BadLineEnd; + + if (numLines_ && ++currentLine == numLines_) + return MissingRegs; + + if (lineLengthBytes_) { + currentOffset = currentLineStart + lineLengthBytes_; + + /* Require whole line to be in the buffer (if buffer size set). */ + if (buffer.size() && + currentOffset + lineLengthBytes_ > buffer.size()) + return MissingRegs; + + if (buffer[currentOffset] != LineStart) + return NoLineStart; + } else { + /* allow a zero line length to mean "hunt for the next line" */ + while (currentOffset < buffer.size() && + buffer[currentOffset] != LineStart) + currentOffset++; + + if (currentOffset == buffer.size()) + return NoLineStart; + } + + /* inc currentOffset to after LineStart */ + currentLineStart = currentOffset++; + } else { + if (tag == RegHiBits) + regNum = (regNum & 0xff) | (dataByte << 8); + else if (tag == RegLowBits) + regNum = (regNum & 0xff00) | dataByte; + else if (tag == RegSkip) + regNum++; + else if (tag == RegValue) { + auto reg = offsets_.find(regNum); + + if (reg != offsets_.end()) { + offsets_[regNum] = currentOffset - 1; + + if (++regsDone == offsets_.size()) + return ParseOk; + } + regNum++; + } else + return IllegalTag; + } + } +} diff --git a/src/ipa/rpi/cam_helper/meson.build b/src/ipa/rpi/cam_helper/meson.build new file mode 100644 index 00000000..bdf2db8e --- /dev/null +++ b/src/ipa/rpi/cam_helper/meson.build @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: CC0-1.0 + +rpi_ipa_cam_helper_sources = files([ + 'cam_helper.cpp', + 'cam_helper_ov5647.cpp', + 'cam_helper_imx219.cpp', + 'cam_helper_imx290.cpp', + 'cam_helper_imx296.cpp', + 'cam_helper_imx477.cpp', + 'cam_helper_imx519.cpp', + 'cam_helper_imx708.cpp', + 'cam_helper_ov9281.cpp', + 'md_parser_smia.cpp', +]) + +rpi_ipa_cam_helper_includes = [ + include_directories('..'), +] + +rpi_ipa_cam_helper_deps = [ + libcamera_private, +] + +rpi_ipa_cam_helper_lib = static_library('rpi_ipa_cam_helper', rpi_ipa_cam_helper_sources, + include_directories : rpi_ipa_cam_helper_includes, + dependencies : rpi_ipa_cam_helper_deps) -- cgit v1.2.1