diff options
Diffstat (limited to 'src/ipa/simple')
-rw-r--r-- | src/ipa/simple/algorithms/agc.cpp | 139 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/agc.h | 33 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/algorithm.h | 22 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/awb.cpp | 69 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/awb.h | 32 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/blc.cpp | 98 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/blc.h | 40 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/lut.cpp | 123 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/lut.h | 40 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/meson.build | 8 | ||||
-rw-r--r-- | src/ipa/simple/data/meson.build | 10 | ||||
-rw-r--r-- | src/ipa/simple/data/uncalibrated.yaml | 10 | ||||
-rw-r--r-- | src/ipa/simple/ipa_context.cpp | 102 | ||||
-rw-r--r-- | src/ipa/simple/ipa_context.h | 77 | ||||
-rw-r--r-- | src/ipa/simple/meson.build | 31 | ||||
-rw-r--r-- | src/ipa/simple/module.h | 30 | ||||
-rw-r--r-- | src/ipa/simple/soft_simple.cpp | 350 |
17 files changed, 1214 insertions, 0 deletions
diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp new file mode 100644 index 00000000..72aade14 --- /dev/null +++ b/src/ipa/simple/algorithms/agc.cpp @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Exposure and gain + */ + +#include "agc.h" + +#include <stdint.h> + +#include <libcamera/base/log.h> + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftExposure) + +namespace ipa::soft::algorithms { + +/* + * The number of bins to use for the optimal exposure calculations. + */ +static constexpr unsigned int kExposureBinsCount = 5; + +/* + * The exposure is optimal when the mean sample value of the histogram is + * in the middle of the range. + */ +static constexpr float kExposureOptimal = kExposureBinsCount / 2.0; + +/* + * This implements the hysteresis for the exposure adjustment. + * It is small enough to have the exposure close to the optimal, and is big + * enough to prevent the exposure from wobbling around the optimal value. + */ +static constexpr float kExposureSatisfactory = 0.2; + +Agc::Agc() +{ +} + +void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) +{ + /* + * kExpDenominator of 10 gives ~10% increment/decrement; + * kExpDenominator of 5 - about ~20% + */ + static constexpr uint8_t kExpDenominator = 10; + static constexpr uint8_t kExpNumeratorUp = kExpDenominator + 1; + static constexpr uint8_t kExpNumeratorDown = kExpDenominator - 1; + + double next; + int32_t &exposure = frameContext.sensor.exposure; + double &again = frameContext.sensor.gain; + + if (exposureMSV < kExposureOptimal - kExposureSatisfactory) { + next = exposure * kExpNumeratorUp / kExpDenominator; + if (next - exposure < 1) + exposure += 1; + else + exposure = next; + if (exposure >= context.configuration.agc.exposureMax) { + next = again * kExpNumeratorUp / kExpDenominator; + if (next - again < context.configuration.agc.againMinStep) + again += context.configuration.agc.againMinStep; + else + again = next; + } + } + + if (exposureMSV > kExposureOptimal + kExposureSatisfactory) { + if (exposure == context.configuration.agc.exposureMax && + again > context.configuration.agc.againMin) { + next = again * kExpNumeratorDown / kExpDenominator; + if (again - next < context.configuration.agc.againMinStep) + again -= context.configuration.agc.againMinStep; + else + again = next; + } else { + next = exposure * kExpNumeratorDown / kExpDenominator; + if (exposure - next < 1) + exposure -= 1; + else + exposure = next; + } + } + + exposure = std::clamp(exposure, context.configuration.agc.exposureMin, + context.configuration.agc.exposureMax); + again = std::clamp(again, context.configuration.agc.againMin, + context.configuration.agc.againMax); + + LOG(IPASoftExposure, Debug) + << "exposureMSV " << exposureMSV + << " exp " << exposure << " again " << again; +} + +void Agc::process(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + const SwIspStats *stats, + [[maybe_unused]] ControlList &metadata) +{ + /* + * Calculate Mean Sample Value (MSV) according to formula from: + * https://www.araa.asn.au/acra/acra2007/papers/paper84final.pdf + */ + const auto &histogram = stats->yHistogram; + const unsigned int blackLevelHistIdx = + context.activeState.blc.level / (256 / SwIspStats::kYHistogramSize); + const unsigned int histogramSize = + SwIspStats::kYHistogramSize - blackLevelHistIdx; + const unsigned int yHistValsPerBin = histogramSize / kExposureBinsCount; + const unsigned int yHistValsPerBinMod = + histogramSize / (histogramSize % kExposureBinsCount + 1); + int exposureBins[kExposureBinsCount] = {}; + unsigned int denom = 0; + unsigned int num = 0; + + for (unsigned int i = 0; i < histogramSize; i++) { + unsigned int idx = (i - (i / yHistValsPerBinMod)) / yHistValsPerBin; + exposureBins[idx] += histogram[blackLevelHistIdx + i]; + } + + for (unsigned int i = 0; i < kExposureBinsCount; i++) { + LOG(IPASoftExposure, Debug) << i << ": " << exposureBins[i]; + denom += exposureBins[i]; + num += exposureBins[i] * (i + 1); + } + + float exposureMSV = (denom == 0 ? 0 : static_cast<float>(num) / denom); + updateExposure(context, frameContext, exposureMSV); +} + +REGISTER_IPA_ALGORITHM(Agc, "Agc") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/agc.h b/src/ipa/simple/algorithms/agc.h new file mode 100644 index 00000000..112d9f5a --- /dev/null +++ b/src/ipa/simple/algorithms/agc.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Exposure and gain + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Agc : public Algorithm +{ +public: + Agc(); + ~Agc() = default; + + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + void updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV); +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/algorithm.h b/src/ipa/simple/algorithms/algorithm.h new file mode 100644 index 00000000..41f63170 --- /dev/null +++ b/src/ipa/simple/algorithms/algorithm.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Red Hat, Inc. + * + * Software ISP control algorithm interface + */ + +#pragma once + +#include <libipa/algorithm.h> + +#include "module.h" + +namespace libcamera { + +namespace ipa::soft { + +using Algorithm = libcamera::ipa::Algorithm<Module>; + +} /* namespace ipa::soft */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/awb.cpp b/src/ipa/simple/algorithms/awb.cpp new file mode 100644 index 00000000..195de41d --- /dev/null +++ b/src/ipa/simple/algorithms/awb.cpp @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Auto white balance + */ + +#include "awb.h" + +#include <numeric> +#include <stdint.h> + +#include <libcamera/base/log.h> + +#include "simple/ipa_context.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftAwb) + +namespace ipa::soft::algorithms { + +int Awb::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + auto &gains = context.activeState.gains; + gains.red = gains.green = gains.blue = 1.0; + + return 0; +} + +void Awb::process(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + const SwIspStats *stats, + [[maybe_unused]] ControlList &metadata) +{ + const SwIspStats::Histogram &histogram = stats->yHistogram; + const uint8_t blackLevel = context.activeState.blc.level; + + /* + * Black level must be subtracted to get the correct AWB ratios, they + * would be off if they were computed from the whole brightness range + * rather than from the sensor range. + */ + const uint64_t nPixels = std::accumulate( + histogram.begin(), histogram.end(), 0); + const uint64_t offset = blackLevel * nPixels; + const uint64_t sumR = stats->sumR_ - offset / 4; + const uint64_t sumG = stats->sumG_ - offset / 2; + const uint64_t sumB = stats->sumB_ - offset / 4; + + /* + * Calculate red and blue gains for AWB. + * Clamp max gain at 4.0, this also avoids 0 division. + */ + auto &gains = context.activeState.gains; + gains.red = sumR <= sumG / 4 ? 4.0 : static_cast<double>(sumG) / sumR; + gains.blue = sumB <= sumG / 4 ? 4.0 : static_cast<double>(sumG) / sumB; + /* Green gain is fixed to 1.0 */ + + LOG(IPASoftAwb, Debug) << "gain R/B " << gains.red << "/" << gains.blue; +} + +REGISTER_IPA_ALGORITHM(Awb, "Awb") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/awb.h b/src/ipa/simple/algorithms/awb.h new file mode 100644 index 00000000..db1496cd --- /dev/null +++ b/src/ipa/simple/algorithms/awb.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Auto white balance + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Awb : public Algorithm +{ +public: + Awb() = default; + ~Awb() = default; + + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void process(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/blc.cpp b/src/ipa/simple/algorithms/blc.cpp new file mode 100644 index 00000000..1d7d370b --- /dev/null +++ b/src/ipa/simple/algorithms/blc.cpp @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Black level handling + */ + +#include "blc.h" + +#include <numeric> + +#include <libcamera/base/log.h> + +namespace libcamera { + +namespace ipa::soft::algorithms { + +LOG_DEFINE_CATEGORY(IPASoftBL) + +BlackLevel::BlackLevel() +{ +} + +int BlackLevel::init([[maybe_unused]] IPAContext &context, + const YamlObject &tuningData) +{ + auto blackLevel = tuningData["blackLevel"].get<int16_t>(); + if (blackLevel.has_value()) { + /* + * Convert 16 bit values from the tuning file to 8 bit black + * level for the SoftISP. + */ + definedLevel_ = blackLevel.value() >> 8; + } + return 0; +} + +int BlackLevel::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + if (definedLevel_.has_value()) + context.configuration.black.level = definedLevel_; + context.activeState.blc.level = + context.configuration.black.level.value_or(255); + return 0; +} + +void BlackLevel::process(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + [[maybe_unused]] ControlList &metadata) +{ + if (context.configuration.black.level.has_value()) + return; + + if (frameContext.sensor.exposure == exposure_ && + frameContext.sensor.gain == gain_) { + return; + } + + const SwIspStats::Histogram &histogram = stats->yHistogram; + + /* + * The constant is selected to be "good enough", not overly + * conservative or aggressive. There is no magic about the given value. + */ + constexpr float ignoredPercentage = 0.02; + const unsigned int total = + std::accumulate(begin(histogram), end(histogram), 0); + const unsigned int pixelThreshold = ignoredPercentage * total; + const unsigned int histogramRatio = 256 / SwIspStats::kYHistogramSize; + const unsigned int currentBlackIdx = + context.activeState.blc.level / histogramRatio; + + for (unsigned int i = 0, seen = 0; + i < currentBlackIdx && i < SwIspStats::kYHistogramSize; + i++) { + seen += histogram[i]; + if (seen >= pixelThreshold) { + context.activeState.blc.level = i * histogramRatio; + exposure_ = frameContext.sensor.exposure; + gain_ = frameContext.sensor.gain; + LOG(IPASoftBL, Debug) + << "Auto-set black level: " + << i << "/" << SwIspStats::kYHistogramSize + << " (" << 100 * (seen - histogram[i]) / total << "% below, " + << 100 * seen / total << "% at or below)"; + break; + } + }; +} + +REGISTER_IPA_ALGORITHM(BlackLevel, "BlackLevel") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/blc.h b/src/ipa/simple/algorithms/blc.h new file mode 100644 index 00000000..52d59cab --- /dev/null +++ b/src/ipa/simple/algorithms/blc.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Black level handling + */ + +#pragma once + +#include <optional> +#include <stdint.h> + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class BlackLevel : public Algorithm +{ +public: + BlackLevel(); + ~BlackLevel() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + int32_t exposure_; + double gain_; + std::optional<uint8_t> definedLevel_; +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/lut.cpp b/src/ipa/simple/algorithms/lut.cpp new file mode 100644 index 00000000..0ba2391f --- /dev/null +++ b/src/ipa/simple/algorithms/lut.cpp @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Color lookup tables construction + */ + +#include "lut.h" + +#include <algorithm> +#include <cmath> +#include <optional> +#include <stdint.h> + +#include <libcamera/base/log.h> + +#include "simple/ipa_context.h" + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftLut) + +namespace ipa::soft::algorithms { + +int Lut::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::Contrast] = ControlInfo(0.0f, 2.0f, 1.0f); + return 0; +} + +int Lut::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + /* Gamma value is fixed */ + context.configuration.gamma = 0.5; + context.activeState.knobs.contrast = std::optional<double>(); + updateGammaTable(context); + + return 0; +} + +void Lut::queueRequest(typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &contrast = controls.get(controls::Contrast); + if (contrast.has_value()) { + context.activeState.knobs.contrast = contrast; + LOG(IPASoftLut, Debug) << "Setting contrast to " << contrast.value(); + } +} + +void Lut::updateGammaTable(IPAContext &context) +{ + auto &gammaTable = context.activeState.gamma.gammaTable; + const auto blackLevel = context.activeState.blc.level; + const unsigned int blackIndex = blackLevel * gammaTable.size() / 256; + const auto contrast = context.activeState.knobs.contrast.value_or(1.0); + + std::fill(gammaTable.begin(), gammaTable.begin() + blackIndex, 0); + const float divisor = gammaTable.size() - blackIndex - 1.0; + for (unsigned int i = blackIndex; i < gammaTable.size(); i++) { + double normalized = (i - blackIndex) / divisor; + /* Convert 0..2 to 0..infinity; avoid actual inifinity at tan(pi/2) */ + double contrastExp = tan(std::clamp(contrast * M_PI_4, 0.0, M_PI_2 - 0.00001)); + /* Apply simple S-curve */ + if (normalized < 0.5) + normalized = 0.5 * std::pow(normalized / 0.5, contrastExp); + else + normalized = 1.0 - 0.5 * std::pow((1.0 - normalized) / 0.5, contrastExp); + gammaTable[i] = UINT8_MAX * + std::pow(normalized, context.configuration.gamma); + } + + context.activeState.gamma.blackLevel = blackLevel; + context.activeState.gamma.contrast = contrast; +} + +void Lut::prepare(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] DebayerParams *params) +{ + /* + * Update the gamma table if needed. This means if black level changes + * and since the black level gets updated only if a lower value is + * observed, it's not permanently prone to minor fluctuations or + * rounding errors. + */ + if (context.activeState.gamma.blackLevel != context.activeState.blc.level || + context.activeState.gamma.contrast != context.activeState.knobs.contrast) + updateGammaTable(context); + + auto &gains = context.activeState.gains; + auto &gammaTable = context.activeState.gamma.gammaTable; + const unsigned int gammaTableSize = gammaTable.size(); + + for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { + const double div = static_cast<double>(DebayerParams::kRGBLookupSize) / + gammaTableSize; + /* Apply gamma after gain! */ + unsigned int idx; + idx = std::min({ static_cast<unsigned int>(i * gains.red / div), + gammaTableSize - 1 }); + params->red[i] = gammaTable[idx]; + idx = std::min({ static_cast<unsigned int>(i * gains.green / div), + gammaTableSize - 1 }); + params->green[i] = gammaTable[idx]; + idx = std::min({ static_cast<unsigned int>(i * gains.blue / div), + gammaTableSize - 1 }); + params->blue[i] = gammaTable[idx]; + } +} + +REGISTER_IPA_ALGORITHM(Lut, "Lut") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/lut.h b/src/ipa/simple/algorithms/lut.h new file mode 100644 index 00000000..889f864b --- /dev/null +++ b/src/ipa/simple/algorithms/lut.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Color lookup tables construction + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Lut : public Algorithm +{ +public: + Lut() = default; + ~Lut() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void prepare(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + DebayerParams *params) override; + +private: + void updateGammaTable(IPAContext &context); +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build new file mode 100644 index 00000000..37a2eb53 --- /dev/null +++ b/src/ipa/simple/algorithms/meson.build @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: CC0-1.0 + +soft_simple_ipa_algorithms = files([ + 'awb.cpp', + 'agc.cpp', + 'blc.cpp', + 'lut.cpp', +]) diff --git a/src/ipa/simple/data/meson.build b/src/ipa/simple/data/meson.build new file mode 100644 index 00000000..92795ee4 --- /dev/null +++ b/src/ipa/simple/data/meson.build @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 + +conf_files = files([ + 'uncalibrated.yaml', +]) + +# The install_dir must match the name from the IPAModuleInfo +install_data(conf_files, + install_dir : ipa_data_dir / 'simple', + install_tag : 'runtime') diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml new file mode 100644 index 00000000..3f147112 --- /dev/null +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 +%YAML 1.1 +--- +version: 1 +algorithms: + - BlackLevel: + - Awb: + - Lut: + - Agc: +... diff --git a/src/ipa/simple/ipa_context.cpp b/src/ipa/simple/ipa_context.cpp new file mode 100644 index 00000000..3f94bbeb --- /dev/null +++ b/src/ipa/simple/ipa_context.cpp @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Google Inc. + * Copyright (C) 2024 Red Hat Inc. + * + * Software ISP IPA Context + */ + +#include "ipa_context.h" + +/** + * \file ipa_context.h + * \brief Context and state information shared between the algorithms + */ + +namespace libcamera::ipa::soft { + +/** + * \struct IPASessionConfiguration + * \brief Session configuration for the IPA module + * + * The session configuration contains all IPA configuration parameters that + * remain constant during the capture session, from IPA module start to stop. + * It is typically set during the configure() operation of the IPA module, but + * may also be updated in the start() operation. + */ + +/** + * \struct IPAActiveState + * \brief The active state of the IPA algorithms + * + * The IPA is fed with the statistics generated from the latest frame processed. + * The statistics are then processed by the IPA algorithms to compute parameters + * required for the next frame capture and processing. The current state of the + * algorithms is reflected through the IPAActiveState to store the values most + * recently computed by the IPA algorithms. + */ + +/** + * \struct IPAContext + * \brief Global IPA context data shared between all algorithms + * + * \var IPAContext::configuration + * \brief The IPA session configuration, immutable during the session + * + * \var IPAContext::frameContexts + * \brief Ring buffer of the IPAFrameContext(s) + * + * \var IPAContext::activeState + * \brief The current state of IPA algorithms + */ + +/** + * \var IPASessionConfiguration::gamma + * \brief Gamma value to be used in the raw image processing + */ + +/** + * \var IPAActiveState::black + * \brief Context for the Black Level algorithm + * + * \var IPAActiveState::black.level + * \brief Current determined black level + */ + +/** + * \var IPAActiveState::gains + * \brief Context for gains in the Colors algorithm + * + * \var IPAActiveState::gains.red + * \brief Gain of red color + * + * \var IPAActiveState::gains.green + * \brief Gain of green color + * + * \var IPAActiveState::gains.blue + * \brief Gain of blue color + */ + +/** + * \var IPAActiveState::agc + * \brief Context for the AGC algorithm + * + * \var IPAActiveState::agc.exposure + * \brief Current exposure value + * + * \var IPAActiveState::agc.again + * \brief Current analog gain value + */ + +/** + * \var IPAActiveState::gamma + * \brief Context for gamma in the Colors algorithm + * + * \var IPAActiveState::gamma.gammaTable + * \brief Computed gamma table + * + * \var IPAActiveState::gamma.blackLevel + * \brief Black level used for the gamma table computation + */ + +} /* namespace libcamera::ipa::soft */ diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h new file mode 100644 index 00000000..4af51306 --- /dev/null +++ b/src/ipa/simple/ipa_context.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Red Hat, Inc. + * + * Simple pipeline IPA Context + */ + +#pragma once + +#include <array> +#include <optional> +#include <stdint.h> + +#include <libcamera/controls.h> + +#include <libipa/fc_queue.h> + +namespace libcamera { + +namespace ipa::soft { + +struct IPASessionConfiguration { + float gamma; + struct { + int32_t exposureMin, exposureMax; + double againMin, againMax, againMinStep; + } agc; + struct { + std::optional<uint8_t> level; + } black; +}; + +struct IPAActiveState { + struct { + uint8_t level; + } blc; + + struct { + double red; + double green; + double blue; + } gains; + + static constexpr unsigned int kGammaLookupSize = 1024; + struct { + std::array<double, kGammaLookupSize> gammaTable; + uint8_t blackLevel; + double contrast; + } gamma; + struct { + /* 0..2 range, 1.0 = normal */ + std::optional<double> contrast; + } knobs; +}; + +struct IPAFrameContext : public FrameContext { + struct { + int32_t exposure; + double gain; + } sensor; +}; + +struct IPAContext { + IPAContext(unsigned int frameContextSize) + : frameContexts(frameContextSize) + { + } + + IPASessionConfiguration configuration; + IPAActiveState activeState; + FCQueue<IPAFrameContext> frameContexts; + ControlInfoMap::Map ctrlMap; +}; + +} /* namespace ipa::soft */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/meson.build b/src/ipa/simple/meson.build new file mode 100644 index 00000000..2f9f15f4 --- /dev/null +++ b/src/ipa/simple/meson.build @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: CC0-1.0 + +subdir('algorithms') +subdir('data') + +ipa_name = 'ipa_soft_simple' + +soft_simple_sources = files([ + 'ipa_context.cpp', + 'soft_simple.cpp', +]) + +soft_simple_sources += soft_simple_ipa_algorithms + +mod = shared_module(ipa_name, soft_simple_sources, + name_prefix : '', + include_directories : [ipa_includes], + dependencies : [libcamera_private, libipa_dep], + install : true, + install_dir : ipa_install_dir) + +if ipa_sign_module + custom_target(ipa_name + '.so.sign', + input : mod, + output : ipa_name + '.so.sign', + command : [ipa_sign, ipa_priv_key, '@INPUT@', '@OUTPUT@'], + install : false, + build_by_default : true) +endif + +ipa_names += ipa_name diff --git a/src/ipa/simple/module.h b/src/ipa/simple/module.h new file mode 100644 index 00000000..8d4d53fb --- /dev/null +++ b/src/ipa/simple/module.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Red Hat, Inc. + * + * Software ISP IPA Module + */ + +#pragma once + +#include <libcamera/controls.h> + +#include <libcamera/ipa/soft_ipa_interface.h> + +#include "libcamera/internal/software_isp/debayer_params.h" +#include "libcamera/internal/software_isp/swisp_stats.h" + +#include <libipa/module.h> + +#include "ipa_context.h" + +namespace libcamera { + +namespace ipa::soft { + +using Module = ipa::Module<IPAContext, IPAFrameContext, IPAConfigInfo, + DebayerParams, SwIspStats>; + +} /* namespace ipa::soft */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/soft_simple.cpp b/src/ipa/simple/soft_simple.cpp new file mode 100644 index 00000000..b26e4e15 --- /dev/null +++ b/src/ipa/simple/soft_simple.cpp @@ -0,0 +1,350 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2023, Linaro Ltd + * + * Simple Software Image Processing Algorithm module + */ + +#include <stdint.h> +#include <sys/mman.h> + +#include <linux/v4l2-controls.h> + +#include <libcamera/base/file.h> +#include <libcamera/base/log.h> +#include <libcamera/base/shared_fd.h> + +#include <libcamera/control_ids.h> +#include <libcamera/controls.h> + +#include <libcamera/ipa/ipa_interface.h> +#include <libcamera/ipa/ipa_module_info.h> +#include <libcamera/ipa/soft_ipa_interface.h> + +#include "libcamera/internal/software_isp/debayer_params.h" +#include "libcamera/internal/software_isp/swisp_stats.h" +#include "libcamera/internal/yaml_parser.h" + +#include "libipa/camera_sensor_helper.h" + +#include "module.h" + +namespace libcamera { +LOG_DEFINE_CATEGORY(IPASoft) + +namespace ipa::soft { + +/* Maximum number of frame contexts to be held */ +static constexpr uint32_t kMaxFrameContexts = 16; + +class IPASoftSimple : public ipa::soft::IPASoftInterface, public Module +{ +public: + IPASoftSimple() + : context_(kMaxFrameContexts) + { + } + + ~IPASoftSimple(); + + int init(const IPASettings &settings, + const SharedFD &fdStats, + const SharedFD &fdParams, + const ControlInfoMap &sensorInfoMap, + ControlInfoMap *ipaControls) override; + int configure(const IPAConfigInfo &configInfo) override; + + int start() override; + void stop() override; + + void queueRequest(const uint32_t frame, const ControlList &controls) override; + void computeParams(const uint32_t frame) override; + void processStats(const uint32_t frame, const uint32_t bufferId, + const ControlList &sensorControls) override; + +protected: + std::string logPrefix() const override; + +private: + void updateExposure(double exposureMSV); + + DebayerParams *params_; + SwIspStats *stats_; + std::unique_ptr<CameraSensorHelper> camHelper_; + ControlInfoMap sensorInfoMap_; + + /* Local parameter storage */ + struct IPAContext context_; +}; + +IPASoftSimple::~IPASoftSimple() +{ + if (stats_) + munmap(stats_, sizeof(SwIspStats)); + if (params_) + munmap(params_, sizeof(DebayerParams)); +} + +int IPASoftSimple::init(const IPASettings &settings, + const SharedFD &fdStats, + const SharedFD &fdParams, + const ControlInfoMap &sensorInfoMap, + ControlInfoMap *ipaControls) +{ + camHelper_ = CameraSensorHelperFactoryBase::create(settings.sensorModel); + if (!camHelper_) { + LOG(IPASoft, Warning) + << "Failed to create camera sensor helper for " + << settings.sensorModel; + } + + /* Load the tuning data file */ + File file(settings.configurationFile); + if (!file.open(File::OpenModeFlag::ReadOnly)) { + int ret = file.error(); + LOG(IPASoft, Error) + << "Failed to open configuration file " + << settings.configurationFile << ": " << strerror(-ret); + return ret; + } + + std::unique_ptr<libcamera::YamlObject> data = YamlParser::parse(file); + if (!data) + return -EINVAL; + + /* \todo Use the IPA configuration file for real. */ + unsigned int version = (*data)["version"].get<uint32_t>(0); + LOG(IPASoft, Debug) << "Tuning file version " << version; + + if (!data->contains("algorithms")) { + LOG(IPASoft, Error) << "Tuning file doesn't contain algorithms"; + return -EINVAL; + } + + int ret = createAlgorithms(context_, (*data)["algorithms"]); + if (ret) + return ret; + + params_ = nullptr; + stats_ = nullptr; + + if (!fdStats.isValid()) { + LOG(IPASoft, Error) << "Invalid Statistics handle"; + return -ENODEV; + } + + if (!fdParams.isValid()) { + LOG(IPASoft, Error) << "Invalid Parameters handle"; + return -ENODEV; + } + + { + void *mem = mmap(nullptr, sizeof(DebayerParams), PROT_WRITE, + MAP_SHARED, fdParams.get(), 0); + if (mem == MAP_FAILED) { + LOG(IPASoft, Error) << "Unable to map Parameters"; + return -errno; + } + + params_ = static_cast<DebayerParams *>(mem); + } + + { + void *mem = mmap(nullptr, sizeof(SwIspStats), PROT_READ, + MAP_SHARED, fdStats.get(), 0); + if (mem == MAP_FAILED) { + LOG(IPASoft, Error) << "Unable to map Statistics"; + return -errno; + } + + stats_ = static_cast<SwIspStats *>(mem); + } + + ControlInfoMap::Map ctrlMap = context_.ctrlMap; + *ipaControls = ControlInfoMap(std::move(ctrlMap), controls::controls); + + /* + * Check if the sensor driver supports the controls required by the + * Soft IPA. + * Don't save the min and max control values yet, as e.g. the limits + * for V4L2_CID_EXPOSURE depend on the configured sensor resolution. + */ + if (sensorInfoMap.find(V4L2_CID_EXPOSURE) == sensorInfoMap.end()) { + LOG(IPASoft, Error) << "Don't have exposure control"; + return -EINVAL; + } + + if (sensorInfoMap.find(V4L2_CID_ANALOGUE_GAIN) == sensorInfoMap.end()) { + LOG(IPASoft, Error) << "Don't have gain control"; + return -EINVAL; + } + + return 0; +} + +int IPASoftSimple::configure(const IPAConfigInfo &configInfo) +{ + sensorInfoMap_ = configInfo.sensorControls; + + const ControlInfo &exposureInfo = sensorInfoMap_.find(V4L2_CID_EXPOSURE)->second; + const ControlInfo &gainInfo = sensorInfoMap_.find(V4L2_CID_ANALOGUE_GAIN)->second; + + /* Clear the IPA context before the streaming session. */ + context_.configuration = {}; + context_.activeState = {}; + context_.frameContexts.clear(); + + context_.configuration.agc.exposureMin = exposureInfo.min().get<int32_t>(); + context_.configuration.agc.exposureMax = exposureInfo.max().get<int32_t>(); + if (!context_.configuration.agc.exposureMin) { + LOG(IPASoft, Warning) << "Minimum exposure is zero, that can't be linear"; + context_.configuration.agc.exposureMin = 1; + } + + int32_t againMin = gainInfo.min().get<int32_t>(); + int32_t againMax = gainInfo.max().get<int32_t>(); + + if (camHelper_) { + context_.configuration.agc.againMin = camHelper_->gain(againMin); + context_.configuration.agc.againMax = camHelper_->gain(againMax); + context_.configuration.agc.againMinStep = + (context_.configuration.agc.againMax - + context_.configuration.agc.againMin) / + 100.0; + if (camHelper_->blackLevel().has_value()) { + /* + * The black level from camHelper_ is a 16 bit value, software ISP + * works with 8 bit pixel values, both regardless of the actual + * sensor pixel width. Hence we obtain the pixel-based black value + * by dividing the value from the helper by 256. + */ + context_.configuration.black.level = + camHelper_->blackLevel().value() / 256; + } + } else { + /* + * The camera sensor gain (g) is usually not equal to the value written + * into the gain register (x). But the way how the AGC algorithm changes + * the gain value to make the total exposure closer to the optimum + * assumes that g(x) is not too far from linear function. If the minimal + * gain is 0, the g(x) is likely to be far from the linear, like + * g(x) = a / (b * x + c). To avoid unexpected changes to the gain by + * the AGC algorithm (abrupt near one edge, and very small near the + * other) we limit the range of the gain values used. + */ + context_.configuration.agc.againMax = againMax; + if (!againMin) { + LOG(IPASoft, Warning) + << "Minimum gain is zero, that can't be linear"; + context_.configuration.agc.againMin = + std::min(100, againMin / 2 + againMax / 2); + } + context_.configuration.agc.againMinStep = 1.0; + } + + for (auto const &algo : algorithms()) { + int ret = algo->configure(context_, configInfo); + if (ret) + return ret; + } + + LOG(IPASoft, Info) + << "Exposure " << context_.configuration.agc.exposureMin << "-" + << context_.configuration.agc.exposureMax + << ", gain " << context_.configuration.agc.againMin << "-" + << context_.configuration.agc.againMax + << " (" << context_.configuration.agc.againMinStep << ")"; + + return 0; +} + +int IPASoftSimple::start() +{ + return 0; +} + +void IPASoftSimple::stop() +{ + context_.frameContexts.clear(); +} + +void IPASoftSimple::queueRequest(const uint32_t frame, const ControlList &controls) +{ + IPAFrameContext &frameContext = context_.frameContexts.alloc(frame); + + for (auto const &algo : algorithms()) + algo->queueRequest(context_, frame, frameContext, controls); +} + +void IPASoftSimple::computeParams(const uint32_t frame) +{ + IPAFrameContext &frameContext = context_.frameContexts.get(frame); + for (auto const &algo : algorithms()) + algo->prepare(context_, frame, frameContext, params_); + setIspParams.emit(); +} + +void IPASoftSimple::processStats(const uint32_t frame, + [[maybe_unused]] const uint32_t bufferId, + const ControlList &sensorControls) +{ + IPAFrameContext &frameContext = context_.frameContexts.get(frame); + + frameContext.sensor.exposure = + sensorControls.get(V4L2_CID_EXPOSURE).get<int32_t>(); + int32_t again = sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get<int32_t>(); + frameContext.sensor.gain = camHelper_ ? camHelper_->gain(again) : again; + + /* + * Software ISP currently does not produce any metadata. Use an empty + * ControlList for now. + * + * \todo Implement proper metadata handling + */ + ControlList metadata(controls::controls); + for (auto const &algo : algorithms()) + algo->process(context_, frame, frameContext, stats_, metadata); + + /* Sanity check */ + if (!sensorControls.contains(V4L2_CID_EXPOSURE) || + !sensorControls.contains(V4L2_CID_ANALOGUE_GAIN)) { + LOG(IPASoft, Error) << "Control(s) missing"; + return; + } + + ControlList ctrls(sensorInfoMap_); + + auto &againNew = frameContext.sensor.gain; + ctrls.set(V4L2_CID_EXPOSURE, frameContext.sensor.exposure); + ctrls.set(V4L2_CID_ANALOGUE_GAIN, + static_cast<int32_t>(camHelper_ ? camHelper_->gainCode(againNew) : againNew)); + + setSensorControls.emit(ctrls); +} + +std::string IPASoftSimple::logPrefix() const +{ + return "IPASoft"; +} + +} /* namespace ipa::soft */ + +/* + * External IPA module interface + */ +extern "C" { +const struct IPAModuleInfo ipaModuleInfo = { + IPA_MODULE_API_VERSION, + 0, + "simple", + "simple", +}; + +IPAInterface *ipaCreate() +{ + return new ipa::soft::IPASoftSimple(); +} + +} /* extern "C" */ + +} /* namespace libcamera */ |