diff options
Diffstat (limited to 'src/ipa/simple')
-rw-r--r-- | src/ipa/simple/algorithms/agc.cpp | 146 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/agc.h | 33 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/algorithm.h | 22 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/awb.cpp | 100 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/awb.h | 36 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/blc.cpp | 107 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/blc.h | 38 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/ccm.cpp | 76 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/ccm.h | 43 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/lut.cpp | 159 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/lut.h | 46 | ||||
-rw-r--r-- | src/ipa/simple/algorithms/meson.build | 9 | ||||
-rw-r--r-- | src/ipa/simple/data/meson.build | 10 | ||||
-rw-r--r-- | src/ipa/simple/data/uncalibrated.yaml | 19 | ||||
-rw-r--r-- | src/ipa/simple/ipa_context.cpp | 102 | ||||
-rw-r--r-- | src/ipa/simple/ipa_context.h | 101 | ||||
-rw-r--r-- | src/ipa/simple/meson.build | 31 | ||||
-rw-r--r-- | src/ipa/simple/module.h | 30 | ||||
-rw-r--r-- | src/ipa/simple/soft_simple.cpp | 358 |
19 files changed, 1466 insertions, 0 deletions
diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp new file mode 100644 index 00000000..c46bb0eb --- /dev/null +++ b/src/ipa/simple/algorithms/agc.cpp @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Exposure and gain + */ + +#include "agc.h" + +#include <stdint.h> + +#include <libcamera/base/log.h> + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftExposure) + +namespace ipa::soft::algorithms { + +/* + * The number of bins to use for the optimal exposure calculations. + */ +static constexpr unsigned int kExposureBinsCount = 5; + +/* + * The exposure is optimal when the mean sample value of the histogram is + * in the middle of the range. + */ +static constexpr float kExposureOptimal = kExposureBinsCount / 2.0; + +/* + * This implements the hysteresis for the exposure adjustment. + * It is small enough to have the exposure close to the optimal, and is big + * enough to prevent the exposure from wobbling around the optimal value. + */ +static constexpr float kExposureSatisfactory = 0.2; + +Agc::Agc() +{ +} + +void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) +{ + /* + * kExpDenominator of 10 gives ~10% increment/decrement; + * kExpDenominator of 5 - about ~20% + */ + static constexpr uint8_t kExpDenominator = 10; + static constexpr uint8_t kExpNumeratorUp = kExpDenominator + 1; + static constexpr uint8_t kExpNumeratorDown = kExpDenominator - 1; + + double next; + int32_t &exposure = frameContext.sensor.exposure; + double &again = frameContext.sensor.gain; + + if (exposureMSV < kExposureOptimal - kExposureSatisfactory) { + next = exposure * kExpNumeratorUp / kExpDenominator; + if (next - exposure < 1) + exposure += 1; + else + exposure = next; + if (exposure >= context.configuration.agc.exposureMax) { + next = again * kExpNumeratorUp / kExpDenominator; + if (next - again < context.configuration.agc.againMinStep) + again += context.configuration.agc.againMinStep; + else + again = next; + } + } + + if (exposureMSV > kExposureOptimal + kExposureSatisfactory) { + if (exposure == context.configuration.agc.exposureMax && + again > context.configuration.agc.againMin) { + next = again * kExpNumeratorDown / kExpDenominator; + if (again - next < context.configuration.agc.againMinStep) + again -= context.configuration.agc.againMinStep; + else + again = next; + } else { + next = exposure * kExpNumeratorDown / kExpDenominator; + if (exposure - next < 1) + exposure -= 1; + else + exposure = next; + } + } + + exposure = std::clamp(exposure, context.configuration.agc.exposureMin, + context.configuration.agc.exposureMax); + again = std::clamp(again, context.configuration.agc.againMin, + context.configuration.agc.againMax); + + LOG(IPASoftExposure, Debug) + << "exposureMSV " << exposureMSV + << " exp " << exposure << " again " << again; +} + +void Agc::process(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) +{ + utils::Duration exposureTime = + context.configuration.agc.lineDuration * frameContext.sensor.exposure; + metadata.set(controls::ExposureTime, exposureTime.get<std::micro>()); + metadata.set(controls::AnalogueGain, frameContext.sensor.gain); + + /* + * Calculate Mean Sample Value (MSV) according to formula from: + * https://www.araa.asn.au/acra/acra2007/papers/paper84final.pdf + */ + const auto &histogram = stats->yHistogram; + const unsigned int blackLevelHistIdx = + context.activeState.blc.level / (256 / SwIspStats::kYHistogramSize); + const unsigned int histogramSize = + SwIspStats::kYHistogramSize - blackLevelHistIdx; + const unsigned int yHistValsPerBin = histogramSize / kExposureBinsCount; + const unsigned int yHistValsPerBinMod = + histogramSize / (histogramSize % kExposureBinsCount + 1); + int exposureBins[kExposureBinsCount] = {}; + unsigned int denom = 0; + unsigned int num = 0; + + for (unsigned int i = 0; i < histogramSize; i++) { + unsigned int idx = (i - (i / yHistValsPerBinMod)) / yHistValsPerBin; + exposureBins[idx] += histogram[blackLevelHistIdx + i]; + } + + for (unsigned int i = 0; i < kExposureBinsCount; i++) { + LOG(IPASoftExposure, Debug) << i << ": " << exposureBins[i]; + denom += exposureBins[i]; + num += exposureBins[i] * (i + 1); + } + + float exposureMSV = (denom == 0 ? 0 : static_cast<float>(num) / denom); + updateExposure(context, frameContext, exposureMSV); +} + +REGISTER_IPA_ALGORITHM(Agc, "Agc") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/agc.h b/src/ipa/simple/algorithms/agc.h new file mode 100644 index 00000000..112d9f5a --- /dev/null +++ b/src/ipa/simple/algorithms/agc.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Exposure and gain + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Agc : public Algorithm +{ +public: + Agc(); + ~Agc() = default; + + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + void updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV); +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/algorithm.h b/src/ipa/simple/algorithms/algorithm.h new file mode 100644 index 00000000..41f63170 --- /dev/null +++ b/src/ipa/simple/algorithms/algorithm.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Red Hat, Inc. + * + * Software ISP control algorithm interface + */ + +#pragma once + +#include <libipa/algorithm.h> + +#include "module.h" + +namespace libcamera { + +namespace ipa::soft { + +using Algorithm = libcamera::ipa::Algorithm<Module>; + +} /* namespace ipa::soft */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/awb.cpp b/src/ipa/simple/algorithms/awb.cpp new file mode 100644 index 00000000..cf567e89 --- /dev/null +++ b/src/ipa/simple/algorithms/awb.cpp @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Auto white balance + */ + +#include "awb.h" + +#include <numeric> +#include <stdint.h> + +#include <libcamera/base/log.h> + +#include <libcamera/control_ids.h> + +#include "libipa/colours.h" +#include "simple/ipa_context.h" + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftAwb) + +namespace ipa::soft::algorithms { + +int Awb::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + auto &gains = context.activeState.awb.gains; + gains = { { 1.0, 1.0, 1.0 } }; + + return 0; +} + +void Awb::prepare(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + [[maybe_unused]] DebayerParams *params) +{ + auto &gains = context.activeState.awb.gains; + /* Just report, the gains are applied in LUT algorithm. */ + frameContext.gains.red = gains.r(); + frameContext.gains.blue = gains.b(); +} + +void Awb::process(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) +{ + const SwIspStats::Histogram &histogram = stats->yHistogram; + const uint8_t blackLevel = context.activeState.blc.level; + + const float maxGain = 1024.0; + const float mdGains[] = { + static_cast<float>(frameContext.gains.red / maxGain), + static_cast<float>(frameContext.gains.blue / maxGain) + }; + metadata.set(controls::ColourGains, mdGains); + + /* + * Black level must be subtracted to get the correct AWB ratios, they + * would be off if they were computed from the whole brightness range + * rather than from the sensor range. + */ + const uint64_t nPixels = std::accumulate( + histogram.begin(), histogram.end(), 0); + const uint64_t offset = blackLevel * nPixels; + const uint64_t sumR = stats->sumR_ - offset / 4; + const uint64_t sumG = stats->sumG_ - offset / 2; + const uint64_t sumB = stats->sumB_ - offset / 4; + + /* + * Calculate red and blue gains for AWB. + * Clamp max gain at 4.0, this also avoids 0 division. + */ + auto &gains = context.activeState.awb.gains; + gains = { { + sumR <= sumG / 4 ? 4.0f : static_cast<float>(sumG) / sumR, + 1.0, + sumB <= sumG / 4 ? 4.0f : static_cast<float>(sumG) / sumB, + } }; + + RGB<double> rgbGains{ { 1 / gains.r(), 1 / gains.g(), 1 / gains.b() } }; + context.activeState.awb.temperatureK = estimateCCT(rgbGains); + metadata.set(controls::ColourTemperature, context.activeState.awb.temperatureK); + + LOG(IPASoftAwb, Debug) + << "gain R/B: " << gains << "; temperature: " + << context.activeState.awb.temperatureK; +} + +REGISTER_IPA_ALGORITHM(Awb, "Awb") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/awb.h b/src/ipa/simple/algorithms/awb.h new file mode 100644 index 00000000..ad993f39 --- /dev/null +++ b/src/ipa/simple/algorithms/awb.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024-2025 Red Hat Inc. + * + * Auto white balance + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Awb : public Algorithm +{ +public: + Awb() = default; + ~Awb() = default; + + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void prepare(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + DebayerParams *params) override; + void process(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/blc.cpp b/src/ipa/simple/algorithms/blc.cpp new file mode 100644 index 00000000..8c1e9ed0 --- /dev/null +++ b/src/ipa/simple/algorithms/blc.cpp @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024-2025, Red Hat Inc. + * + * Black level handling + */ + +#include "blc.h" + +#include <numeric> + +#include <libcamera/base/log.h> + +#include "control_ids.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +LOG_DEFINE_CATEGORY(IPASoftBL) + +BlackLevel::BlackLevel() +{ +} + +int BlackLevel::init([[maybe_unused]] IPAContext &context, + const YamlObject &tuningData) +{ + auto blackLevel = tuningData["blackLevel"].get<int16_t>(); + if (blackLevel.has_value()) { + /* + * Convert 16 bit values from the tuning file to 8 bit black + * level for the SoftISP. + */ + definedLevel_ = blackLevel.value() >> 8; + } + return 0; +} + +int BlackLevel::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + if (definedLevel_.has_value()) + context.configuration.black.level = definedLevel_; + context.activeState.blc.level = + context.configuration.black.level.value_or(255); + return 0; +} + +void BlackLevel::process(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) +{ + /* Assign each of the R G G B channels as the same black level. */ + const int32_t blackLevel = context.activeState.blc.level * 256; + const int32_t blackLevels[] = { + blackLevel, blackLevel, blackLevel, blackLevel + }; + metadata.set(controls::SensorBlackLevels, blackLevels); + + if (context.configuration.black.level.has_value()) + return; + + if (frameContext.sensor.exposure == context.activeState.blc.lastExposure && + frameContext.sensor.gain == context.activeState.blc.lastGain) { + return; + } + + const SwIspStats::Histogram &histogram = stats->yHistogram; + + /* + * The constant is selected to be "good enough", not overly + * conservative or aggressive. There is no magic about the given value. + */ + constexpr float ignoredPercentage = 0.02; + const unsigned int total = + std::accumulate(begin(histogram), end(histogram), 0); + const unsigned int pixelThreshold = ignoredPercentage * total; + const unsigned int histogramRatio = 256 / SwIspStats::kYHistogramSize; + const unsigned int currentBlackIdx = + context.activeState.blc.level / histogramRatio; + + for (unsigned int i = 0, seen = 0; + i < currentBlackIdx && i < SwIspStats::kYHistogramSize; + i++) { + seen += histogram[i]; + if (seen >= pixelThreshold) { + context.activeState.blc.level = i * histogramRatio; + context.activeState.blc.lastExposure = frameContext.sensor.exposure; + context.activeState.blc.lastGain = frameContext.sensor.gain; + LOG(IPASoftBL, Debug) + << "Auto-set black level: " + << i << "/" << SwIspStats::kYHistogramSize + << " (" << 100 * (seen - histogram[i]) / total << "% below, " + << 100 * seen / total << "% at or below)"; + break; + } + }; +} + +REGISTER_IPA_ALGORITHM(BlackLevel, "BlackLevel") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/blc.h b/src/ipa/simple/algorithms/blc.h new file mode 100644 index 00000000..db9e6d63 --- /dev/null +++ b/src/ipa/simple/algorithms/blc.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Black level handling + */ + +#pragma once + +#include <optional> +#include <stdint.h> + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class BlackLevel : public Algorithm +{ +public: + BlackLevel(); + ~BlackLevel() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + std::optional<uint8_t> definedLevel_; +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/ccm.cpp b/src/ipa/simple/algorithms/ccm.cpp new file mode 100644 index 00000000..d5ba928d --- /dev/null +++ b/src/ipa/simple/algorithms/ccm.cpp @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Ideas On Board + * Copyright (C) 2024-2025, Red Hat Inc. + * + * Color correction matrix + */ + +#include "ccm.h" + +#include <libcamera/base/log.h> +#include <libcamera/base/utils.h> + +#include <libcamera/control_ids.h> + +namespace { + +constexpr unsigned int kTemperatureThreshold = 100; + +} + +namespace libcamera { + +namespace ipa::soft::algorithms { + +LOG_DEFINE_CATEGORY(IPASoftCcm) + +int Ccm::init([[maybe_unused]] IPAContext &context, const YamlObject &tuningData) +{ + int ret = ccm_.readYaml(tuningData["ccms"], "ct", "ccm"); + if (ret < 0) { + LOG(IPASoftCcm, Error) + << "Failed to parse 'ccm' parameter from tuning file."; + return ret; + } + + context.ccmEnabled = true; + + return 0; +} + +void Ccm::prepare(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, [[maybe_unused]] DebayerParams *params) +{ + const unsigned int ct = context.activeState.awb.temperatureK; + + /* Change CCM only on bigger temperature changes. */ + if (frame > 0 && + utils::abs_diff(ct, lastCt_) < kTemperatureThreshold) { + frameContext.ccm.ccm = context.activeState.ccm.ccm; + context.activeState.ccm.changed = false; + return; + } + + lastCt_ = ct; + Matrix<float, 3, 3> ccm = ccm_.getInterpolated(ct); + + context.activeState.ccm.ccm = ccm; + frameContext.ccm.ccm = ccm; + context.activeState.ccm.changed = true; +} + +void Ccm::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + [[maybe_unused]] const SwIspStats *stats, + ControlList &metadata) +{ + metadata.set(controls::ColourCorrectionMatrix, frameContext.ccm.ccm.data()); +} + +REGISTER_IPA_ALGORITHM(Ccm, "Ccm") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/ccm.h b/src/ipa/simple/algorithms/ccm.h new file mode 100644 index 00000000..f4e2b85b --- /dev/null +++ b/src/ipa/simple/algorithms/ccm.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024-2025, Red Hat Inc. + * + * Color correction matrix + */ + +#pragma once + +#include "libcamera/internal/matrix.h" + +#include <libipa/interpolator.h> + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Ccm : public Algorithm +{ +public: + Ccm() = default; + ~Ccm() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + void prepare(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + DebayerParams *params) override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + unsigned int lastCt_; + Interpolator<Matrix<float, 3, 3>> ccm_; +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/lut.cpp b/src/ipa/simple/algorithms/lut.cpp new file mode 100644 index 00000000..d1d5f727 --- /dev/null +++ b/src/ipa/simple/algorithms/lut.cpp @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024-2025, Red Hat Inc. + * + * Color lookup tables construction + */ + +#include "lut.h" + +#include <algorithm> +#include <cmath> +#include <optional> +#include <stdint.h> + +#include <libcamera/base/log.h> + +#include <libcamera/control_ids.h> + +#include "simple/ipa_context.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftLut) + +namespace ipa::soft::algorithms { + +int Lut::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::Contrast] = ControlInfo(0.0f, 2.0f, 1.0f); + return 0; +} + +int Lut::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + /* Gamma value is fixed */ + context.configuration.gamma = 0.5; + context.activeState.knobs.contrast = std::optional<double>(); + updateGammaTable(context); + + return 0; +} + +void Lut::queueRequest(typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &contrast = controls.get(controls::Contrast); + if (contrast.has_value()) { + context.activeState.knobs.contrast = contrast; + LOG(IPASoftLut, Debug) << "Setting contrast to " << contrast.value(); + } +} + +void Lut::updateGammaTable(IPAContext &context) +{ + auto &gammaTable = context.activeState.gamma.gammaTable; + const auto blackLevel = context.activeState.blc.level; + const unsigned int blackIndex = blackLevel * gammaTable.size() / 256; + const auto contrast = context.activeState.knobs.contrast.value_or(1.0); + + std::fill(gammaTable.begin(), gammaTable.begin() + blackIndex, 0); + const float divisor = gammaTable.size() - blackIndex - 1.0; + for (unsigned int i = blackIndex; i < gammaTable.size(); i++) { + double normalized = (i - blackIndex) / divisor; + /* Convert 0..2 to 0..infinity; avoid actual inifinity at tan(pi/2) */ + double contrastExp = tan(std::clamp(contrast * M_PI_4, 0.0, M_PI_2 - 0.00001)); + /* Apply simple S-curve */ + if (normalized < 0.5) + normalized = 0.5 * std::pow(normalized / 0.5, contrastExp); + else + normalized = 1.0 - 0.5 * std::pow((1.0 - normalized) / 0.5, contrastExp); + gammaTable[i] = UINT8_MAX * + std::pow(normalized, context.configuration.gamma); + } + + context.activeState.gamma.blackLevel = blackLevel; + context.activeState.gamma.contrast = contrast; +} + +int16_t Lut::ccmValue(unsigned int i, float ccm) const +{ + return std::round(i * ccm); +} + +void Lut::prepare(IPAContext &context, + [[maybe_unused]] const uint32_t frame, + IPAFrameContext &frameContext, + DebayerParams *params) +{ + frameContext.contrast = context.activeState.knobs.contrast; + + /* + * Update the gamma table if needed. This means if black level changes + * and since the black level gets updated only if a lower value is + * observed, it's not permanently prone to minor fluctuations or + * rounding errors. + */ + const bool gammaUpdateNeeded = + context.activeState.gamma.blackLevel != context.activeState.blc.level || + context.activeState.gamma.contrast != context.activeState.knobs.contrast; + if (gammaUpdateNeeded) + updateGammaTable(context); + + auto &gains = context.activeState.awb.gains; + auto &gammaTable = context.activeState.gamma.gammaTable; + const unsigned int gammaTableSize = gammaTable.size(); + const double div = static_cast<double>(DebayerParams::kRGBLookupSize) / + gammaTableSize; + + if (!context.ccmEnabled) { + for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { + /* Apply gamma after gain! */ + const RGB<float> lutGains = (gains * i / div).min(gammaTableSize - 1); + params->red[i] = gammaTable[static_cast<unsigned int>(lutGains.r())]; + params->green[i] = gammaTable[static_cast<unsigned int>(lutGains.g())]; + params->blue[i] = gammaTable[static_cast<unsigned int>(lutGains.b())]; + } + } else if (context.activeState.ccm.changed || gammaUpdateNeeded) { + Matrix<float, 3, 3> gainCcm = { { gains.r(), 0, 0, + 0, gains.g(), 0, + 0, 0, gains.b() } }; + auto ccm = context.activeState.ccm.ccm * gainCcm; + auto &red = params->redCcm; + auto &green = params->greenCcm; + auto &blue = params->blueCcm; + for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { + red[i].r = ccmValue(i, ccm[0][0]); + red[i].g = ccmValue(i, ccm[1][0]); + red[i].b = ccmValue(i, ccm[2][0]); + green[i].r = ccmValue(i, ccm[0][1]); + green[i].g = ccmValue(i, ccm[1][1]); + green[i].b = ccmValue(i, ccm[2][1]); + blue[i].r = ccmValue(i, ccm[0][2]); + blue[i].g = ccmValue(i, ccm[1][2]); + blue[i].b = ccmValue(i, ccm[2][2]); + params->gammaLut[i] = gammaTable[i / div]; + } + } +} + +void Lut::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const SwIspStats *stats, + ControlList &metadata) +{ + const auto &contrast = frameContext.contrast; + if (contrast) + metadata.set(controls::Contrast, contrast.value()); +} + +REGISTER_IPA_ALGORITHM(Lut, "Lut") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/lut.h b/src/ipa/simple/algorithms/lut.h new file mode 100644 index 00000000..ba8b9021 --- /dev/null +++ b/src/ipa/simple/algorithms/lut.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Color lookup tables construction + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Lut : public Algorithm +{ +public: + Lut() = default; + ~Lut() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void prepare(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + DebayerParams *params) override; + void process(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + void updateGammaTable(IPAContext &context); + int16_t ccmValue(unsigned int i, float ccm) const; +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build new file mode 100644 index 00000000..2d0adb05 --- /dev/null +++ b/src/ipa/simple/algorithms/meson.build @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: CC0-1.0 + +soft_simple_ipa_algorithms = files([ + 'awb.cpp', + 'agc.cpp', + 'blc.cpp', + 'ccm.cpp', + 'lut.cpp', +]) diff --git a/src/ipa/simple/data/meson.build b/src/ipa/simple/data/meson.build new file mode 100644 index 00000000..92795ee4 --- /dev/null +++ b/src/ipa/simple/data/meson.build @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 + +conf_files = files([ + 'uncalibrated.yaml', +]) + +# The install_dir must match the name from the IPAModuleInfo +install_data(conf_files, + install_dir : ipa_data_dir / 'simple', + install_tag : 'runtime') diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml new file mode 100644 index 00000000..5508e668 --- /dev/null +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: CC0-1.0 +%YAML 1.1 +--- +version: 1 +algorithms: + - BlackLevel: + - Awb: + # Color correction matrices can be defined here. The CCM algorithm + # has a significant performance impact, and should only be enabled + # if tuned. + # - Ccm: + # ccms: + # - ct: 6500 + # ccm: [ 1, 0, 0, + # 0, 1, 0, + # 0, 0, 1] + - Lut: + - Agc: +... diff --git a/src/ipa/simple/ipa_context.cpp b/src/ipa/simple/ipa_context.cpp new file mode 100644 index 00000000..3f94bbeb --- /dev/null +++ b/src/ipa/simple/ipa_context.cpp @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Google Inc. + * Copyright (C) 2024 Red Hat Inc. + * + * Software ISP IPA Context + */ + +#include "ipa_context.h" + +/** + * \file ipa_context.h + * \brief Context and state information shared between the algorithms + */ + +namespace libcamera::ipa::soft { + +/** + * \struct IPASessionConfiguration + * \brief Session configuration for the IPA module + * + * The session configuration contains all IPA configuration parameters that + * remain constant during the capture session, from IPA module start to stop. + * It is typically set during the configure() operation of the IPA module, but + * may also be updated in the start() operation. + */ + +/** + * \struct IPAActiveState + * \brief The active state of the IPA algorithms + * + * The IPA is fed with the statistics generated from the latest frame processed. + * The statistics are then processed by the IPA algorithms to compute parameters + * required for the next frame capture and processing. The current state of the + * algorithms is reflected through the IPAActiveState to store the values most + * recently computed by the IPA algorithms. + */ + +/** + * \struct IPAContext + * \brief Global IPA context data shared between all algorithms + * + * \var IPAContext::configuration + * \brief The IPA session configuration, immutable during the session + * + * \var IPAContext::frameContexts + * \brief Ring buffer of the IPAFrameContext(s) + * + * \var IPAContext::activeState + * \brief The current state of IPA algorithms + */ + +/** + * \var IPASessionConfiguration::gamma + * \brief Gamma value to be used in the raw image processing + */ + +/** + * \var IPAActiveState::black + * \brief Context for the Black Level algorithm + * + * \var IPAActiveState::black.level + * \brief Current determined black level + */ + +/** + * \var IPAActiveState::gains + * \brief Context for gains in the Colors algorithm + * + * \var IPAActiveState::gains.red + * \brief Gain of red color + * + * \var IPAActiveState::gains.green + * \brief Gain of green color + * + * \var IPAActiveState::gains.blue + * \brief Gain of blue color + */ + +/** + * \var IPAActiveState::agc + * \brief Context for the AGC algorithm + * + * \var IPAActiveState::agc.exposure + * \brief Current exposure value + * + * \var IPAActiveState::agc.again + * \brief Current analog gain value + */ + +/** + * \var IPAActiveState::gamma + * \brief Context for gamma in the Colors algorithm + * + * \var IPAActiveState::gamma.gammaTable + * \brief Computed gamma table + * + * \var IPAActiveState::gamma.blackLevel + * \brief Black level used for the gamma table computation + */ + +} /* namespace libcamera::ipa::soft */ diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h new file mode 100644 index 00000000..88cc6c35 --- /dev/null +++ b/src/ipa/simple/ipa_context.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024-2025 Red Hat, Inc. + * + * Simple pipeline IPA Context + */ + +#pragma once + +#include <array> +#include <optional> +#include <stdint.h> + +#include <libcamera/controls.h> + +#include "libcamera/internal/matrix.h" +#include "libcamera/internal/vector.h" + +#include <libipa/fc_queue.h> + +#include "core_ipa_interface.h" + +namespace libcamera { + +namespace ipa::soft { + +struct IPASessionConfiguration { + float gamma; + struct { + int32_t exposureMin, exposureMax; + double againMin, againMax, againMinStep; + utils::Duration lineDuration; + } agc; + struct { + std::optional<uint8_t> level; + } black; +}; + +struct IPAActiveState { + struct { + uint8_t level; + int32_t lastExposure; + double lastGain; + } blc; + + struct { + RGB<float> gains; + unsigned int temperatureK; + } awb; + + static constexpr unsigned int kGammaLookupSize = 1024; + struct { + std::array<double, kGammaLookupSize> gammaTable; + uint8_t blackLevel; + double contrast; + } gamma; + + struct { + Matrix<float, 3, 3> ccm; + bool changed; + } ccm; + + struct { + /* 0..2 range, 1.0 = normal */ + std::optional<double> contrast; + } knobs; +}; + +struct IPAFrameContext : public FrameContext { + struct { + Matrix<float, 3, 3> ccm; + } ccm; + + struct { + int32_t exposure; + double gain; + } sensor; + struct { + double red; + double blue; + } gains; + std::optional<double> contrast; +}; + +struct IPAContext { + IPAContext(unsigned int frameContextSize) + : frameContexts(frameContextSize) + { + } + + IPACameraSensorInfo sensorInfo; + IPASessionConfiguration configuration; + IPAActiveState activeState; + FCQueue<IPAFrameContext> frameContexts; + ControlInfoMap::Map ctrlMap; + bool ccmEnabled = false; +}; + +} /* namespace ipa::soft */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/meson.build b/src/ipa/simple/meson.build new file mode 100644 index 00000000..2f9f15f4 --- /dev/null +++ b/src/ipa/simple/meson.build @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: CC0-1.0 + +subdir('algorithms') +subdir('data') + +ipa_name = 'ipa_soft_simple' + +soft_simple_sources = files([ + 'ipa_context.cpp', + 'soft_simple.cpp', +]) + +soft_simple_sources += soft_simple_ipa_algorithms + +mod = shared_module(ipa_name, soft_simple_sources, + name_prefix : '', + include_directories : [ipa_includes], + dependencies : [libcamera_private, libipa_dep], + install : true, + install_dir : ipa_install_dir) + +if ipa_sign_module + custom_target(ipa_name + '.so.sign', + input : mod, + output : ipa_name + '.so.sign', + command : [ipa_sign, ipa_priv_key, '@INPUT@', '@OUTPUT@'], + install : false, + build_by_default : true) +endif + +ipa_names += ipa_name diff --git a/src/ipa/simple/module.h b/src/ipa/simple/module.h new file mode 100644 index 00000000..8d4d53fb --- /dev/null +++ b/src/ipa/simple/module.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024 Red Hat, Inc. + * + * Software ISP IPA Module + */ + +#pragma once + +#include <libcamera/controls.h> + +#include <libcamera/ipa/soft_ipa_interface.h> + +#include "libcamera/internal/software_isp/debayer_params.h" +#include "libcamera/internal/software_isp/swisp_stats.h" + +#include <libipa/module.h> + +#include "ipa_context.h" + +namespace libcamera { + +namespace ipa::soft { + +using Module = ipa::Module<IPAContext, IPAFrameContext, IPAConfigInfo, + DebayerParams, SwIspStats>; + +} /* namespace ipa::soft */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/soft_simple.cpp b/src/ipa/simple/soft_simple.cpp new file mode 100644 index 00000000..c94c4cd5 --- /dev/null +++ b/src/ipa/simple/soft_simple.cpp @@ -0,0 +1,358 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2023, Linaro Ltd + * + * Simple Software Image Processing Algorithm module + */ + +#include <chrono> +#include <stdint.h> +#include <sys/mman.h> + +#include <linux/v4l2-controls.h> + +#include <libcamera/base/file.h> +#include <libcamera/base/log.h> +#include <libcamera/base/shared_fd.h> + +#include <libcamera/control_ids.h> +#include <libcamera/controls.h> + +#include <libcamera/ipa/ipa_interface.h> +#include <libcamera/ipa/ipa_module_info.h> +#include <libcamera/ipa/soft_ipa_interface.h> + +#include "libcamera/internal/software_isp/debayer_params.h" +#include "libcamera/internal/software_isp/swisp_stats.h" +#include "libcamera/internal/yaml_parser.h" + +#include "libipa/camera_sensor_helper.h" + +#include "module.h" + +namespace libcamera { +LOG_DEFINE_CATEGORY(IPASoft) + +using namespace std::literals::chrono_literals; + +namespace ipa::soft { + +/* Maximum number of frame contexts to be held */ +static constexpr uint32_t kMaxFrameContexts = 16; + +class IPASoftSimple : public ipa::soft::IPASoftInterface, public Module +{ +public: + IPASoftSimple() + : context_(kMaxFrameContexts) + { + } + + ~IPASoftSimple(); + + int init(const IPASettings &settings, + const SharedFD &fdStats, + const SharedFD &fdParams, + const IPACameraSensorInfo &sensorInfo, + const ControlInfoMap &sensorControls, + ControlInfoMap *ipaControls, + bool *ccmEnabled) override; + int configure(const IPAConfigInfo &configInfo) override; + + int start() override; + void stop() override; + + void queueRequest(const uint32_t frame, const ControlList &controls) override; + void computeParams(const uint32_t frame) override; + void processStats(const uint32_t frame, const uint32_t bufferId, + const ControlList &sensorControls) override; + +protected: + std::string logPrefix() const override; + +private: + void updateExposure(double exposureMSV); + + DebayerParams *params_; + SwIspStats *stats_; + std::unique_ptr<CameraSensorHelper> camHelper_; + ControlInfoMap sensorInfoMap_; + + /* Local parameter storage */ + struct IPAContext context_; +}; + +IPASoftSimple::~IPASoftSimple() +{ + if (stats_) + munmap(stats_, sizeof(SwIspStats)); + if (params_) + munmap(params_, sizeof(DebayerParams)); +} + +int IPASoftSimple::init(const IPASettings &settings, + const SharedFD &fdStats, + const SharedFD &fdParams, + const IPACameraSensorInfo &sensorInfo, + const ControlInfoMap &sensorControls, + ControlInfoMap *ipaControls, + bool *ccmEnabled) +{ + camHelper_ = CameraSensorHelperFactoryBase::create(settings.sensorModel); + if (!camHelper_) { + LOG(IPASoft, Warning) + << "Failed to create camera sensor helper for " + << settings.sensorModel; + } + + context_.sensorInfo = sensorInfo; + + /* Load the tuning data file */ + File file(settings.configurationFile); + if (!file.open(File::OpenModeFlag::ReadOnly)) { + int ret = file.error(); + LOG(IPASoft, Error) + << "Failed to open configuration file " + << settings.configurationFile << ": " << strerror(-ret); + return ret; + } + + std::unique_ptr<libcamera::YamlObject> data = YamlParser::parse(file); + if (!data) + return -EINVAL; + + /* \todo Use the IPA configuration file for real. */ + unsigned int version = (*data)["version"].get<uint32_t>(0); + LOG(IPASoft, Debug) << "Tuning file version " << version; + + if (!data->contains("algorithms")) { + LOG(IPASoft, Error) << "Tuning file doesn't contain algorithms"; + return -EINVAL; + } + + int ret = createAlgorithms(context_, (*data)["algorithms"]); + if (ret) + return ret; + + *ccmEnabled = context_.ccmEnabled; + + params_ = nullptr; + stats_ = nullptr; + + if (!fdStats.isValid()) { + LOG(IPASoft, Error) << "Invalid Statistics handle"; + return -ENODEV; + } + + if (!fdParams.isValid()) { + LOG(IPASoft, Error) << "Invalid Parameters handle"; + return -ENODEV; + } + + { + void *mem = mmap(nullptr, sizeof(DebayerParams), PROT_WRITE, + MAP_SHARED, fdParams.get(), 0); + if (mem == MAP_FAILED) { + LOG(IPASoft, Error) << "Unable to map Parameters"; + return -errno; + } + + params_ = static_cast<DebayerParams *>(mem); + } + + { + void *mem = mmap(nullptr, sizeof(SwIspStats), PROT_READ, + MAP_SHARED, fdStats.get(), 0); + if (mem == MAP_FAILED) { + LOG(IPASoft, Error) << "Unable to map Statistics"; + return -errno; + } + + stats_ = static_cast<SwIspStats *>(mem); + } + + ControlInfoMap::Map ctrlMap = context_.ctrlMap; + *ipaControls = ControlInfoMap(std::move(ctrlMap), controls::controls); + + /* + * Check if the sensor driver supports the controls required by the + * Soft IPA. + * Don't save the min and max control values yet, as e.g. the limits + * for V4L2_CID_EXPOSURE depend on the configured sensor resolution. + */ + if (sensorControls.find(V4L2_CID_EXPOSURE) == sensorControls.end()) { + LOG(IPASoft, Error) << "Don't have exposure control"; + return -EINVAL; + } + + if (sensorControls.find(V4L2_CID_ANALOGUE_GAIN) == sensorControls.end()) { + LOG(IPASoft, Error) << "Don't have gain control"; + return -EINVAL; + } + + return 0; +} + +int IPASoftSimple::configure(const IPAConfigInfo &configInfo) +{ + sensorInfoMap_ = configInfo.sensorControls; + + const ControlInfo &exposureInfo = sensorInfoMap_.find(V4L2_CID_EXPOSURE)->second; + const ControlInfo &gainInfo = sensorInfoMap_.find(V4L2_CID_ANALOGUE_GAIN)->second; + + /* Clear the IPA context before the streaming session. */ + context_.configuration = {}; + context_.activeState = {}; + context_.frameContexts.clear(); + + context_.configuration.agc.lineDuration = + context_.sensorInfo.minLineLength * 1.0s / context_.sensorInfo.pixelRate; + context_.configuration.agc.exposureMin = exposureInfo.min().get<int32_t>(); + context_.configuration.agc.exposureMax = exposureInfo.max().get<int32_t>(); + if (!context_.configuration.agc.exposureMin) { + LOG(IPASoft, Warning) << "Minimum exposure is zero, that can't be linear"; + context_.configuration.agc.exposureMin = 1; + } + + int32_t againMin = gainInfo.min().get<int32_t>(); + int32_t againMax = gainInfo.max().get<int32_t>(); + + if (camHelper_) { + context_.configuration.agc.againMin = camHelper_->gain(againMin); + context_.configuration.agc.againMax = camHelper_->gain(againMax); + context_.configuration.agc.againMinStep = + (context_.configuration.agc.againMax - + context_.configuration.agc.againMin) / + 100.0; + if (camHelper_->blackLevel().has_value()) { + /* + * The black level from camHelper_ is a 16 bit value, software ISP + * works with 8 bit pixel values, both regardless of the actual + * sensor pixel width. Hence we obtain the pixel-based black value + * by dividing the value from the helper by 256. + */ + context_.configuration.black.level = + camHelper_->blackLevel().value() / 256; + } + } else { + /* + * The camera sensor gain (g) is usually not equal to the value written + * into the gain register (x). But the way how the AGC algorithm changes + * the gain value to make the total exposure closer to the optimum + * assumes that g(x) is not too far from linear function. If the minimal + * gain is 0, the g(x) is likely to be far from the linear, like + * g(x) = a / (b * x + c). To avoid unexpected changes to the gain by + * the AGC algorithm (abrupt near one edge, and very small near the + * other) we limit the range of the gain values used. + */ + context_.configuration.agc.againMax = againMax; + if (!againMin) { + LOG(IPASoft, Warning) + << "Minimum gain is zero, that can't be linear"; + context_.configuration.agc.againMin = + std::min(100, againMin / 2 + againMax / 2); + } + context_.configuration.agc.againMinStep = 1.0; + } + + for (auto const &algo : algorithms()) { + int ret = algo->configure(context_, configInfo); + if (ret) + return ret; + } + + LOG(IPASoft, Info) + << "Exposure " << context_.configuration.agc.exposureMin << "-" + << context_.configuration.agc.exposureMax + << ", gain " << context_.configuration.agc.againMin << "-" + << context_.configuration.agc.againMax + << " (" << context_.configuration.agc.againMinStep << ")"; + + return 0; +} + +int IPASoftSimple::start() +{ + return 0; +} + +void IPASoftSimple::stop() +{ + context_.frameContexts.clear(); +} + +void IPASoftSimple::queueRequest(const uint32_t frame, const ControlList &controls) +{ + IPAFrameContext &frameContext = context_.frameContexts.alloc(frame); + + for (auto const &algo : algorithms()) + algo->queueRequest(context_, frame, frameContext, controls); +} + +void IPASoftSimple::computeParams(const uint32_t frame) +{ + IPAFrameContext &frameContext = context_.frameContexts.get(frame); + for (auto const &algo : algorithms()) + algo->prepare(context_, frame, frameContext, params_); + setIspParams.emit(); +} + +void IPASoftSimple::processStats(const uint32_t frame, + [[maybe_unused]] const uint32_t bufferId, + const ControlList &sensorControls) +{ + IPAFrameContext &frameContext = context_.frameContexts.get(frame); + + frameContext.sensor.exposure = + sensorControls.get(V4L2_CID_EXPOSURE).get<int32_t>(); + int32_t again = sensorControls.get(V4L2_CID_ANALOGUE_GAIN).get<int32_t>(); + frameContext.sensor.gain = camHelper_ ? camHelper_->gain(again) : again; + + ControlList metadata(controls::controls); + for (auto const &algo : algorithms()) + algo->process(context_, frame, frameContext, stats_, metadata); + metadataReady.emit(frame, metadata); + + /* Sanity check */ + if (!sensorControls.contains(V4L2_CID_EXPOSURE) || + !sensorControls.contains(V4L2_CID_ANALOGUE_GAIN)) { + LOG(IPASoft, Error) << "Control(s) missing"; + return; + } + + ControlList ctrls(sensorInfoMap_); + + auto &againNew = frameContext.sensor.gain; + ctrls.set(V4L2_CID_EXPOSURE, frameContext.sensor.exposure); + ctrls.set(V4L2_CID_ANALOGUE_GAIN, + static_cast<int32_t>(camHelper_ ? camHelper_->gainCode(againNew) : againNew)); + + setSensorControls.emit(ctrls); +} + +std::string IPASoftSimple::logPrefix() const +{ + return "IPASoft"; +} + +} /* namespace ipa::soft */ + +/* + * External IPA module interface + */ +extern "C" { +const struct IPAModuleInfo ipaModuleInfo = { + IPA_MODULE_API_VERSION, + 0, + "simple", + "simple", +}; + +IPAInterface *ipaCreate() +{ + return new ipa::soft::IPASoftSimple(); +} + +} /* extern "C" */ + +} /* namespace libcamera */ |