/* SPDX-License-Identifier: Apache-2.0 */ /* * Copyright (C) 2020, Google Inc. * * ipu3.cpp - IPU3 Image Processing Algorithms */ #include #include #include #include #include #include #include #include #include #include #include #include #include "libcamera-helpers/mapped_buffer.h" /* IA AIQ Wrapper API */ #include "aic/aic.h" #include "aiq/aiq.h" #include "binary_data.h" namespace libcamera { LOG_DEFINE_CATEGORY(IPAIPU3) namespace ipa::ipu3 { class IPAIPU3 : public IPAIPU3Interface { public: int init(const IPASettings &settings) override; int start() override; void stop() override {} int configure(const IPAConfigInfo &configInfo) override; void mapBuffers(const std::vector &buffers) override; void unmapBuffers(const std::vector &ids) override; void processEvent(const IPU3Event &event) override; private: void processControls(unsigned int frame, const ControlList &metadata); void fillParams(unsigned int frame, ipu3_uapi_params *params); void parseStatistics(unsigned int frame, int64_t frameTimestamp, const ipu3_uapi_stats_3a *stats); void setControls(unsigned int frame); std::map buffers_; ControlInfoMap ctrls_; IPACameraSensorInfo sensorInfo_; /* Camera sensor controls. */ uint32_t exposure_; uint32_t minExposure_; uint32_t maxExposure_; uint32_t gain_; uint32_t minGain_; uint32_t maxGain_; /* Intel Library Instances. */ aiq::AIQ aiq_; aic::AIC aic_; /* Temporary storage until we have a FrameContext object / struct */ aiq::AiqInputParameters aiqInputParams_; aiq::AiqResults results_; BinaryData aiqb_; BinaryData nvm_; BinaryData aiqd_; uint64_t lastFrameTimestamp_; }; int IPAIPU3::init(const IPASettings &settings) { int ret; logSetFile("/tmp/ipu3ipa.log"); /* * Temporary mapping of the sensor name to the AIQB data file. * * \todo: This mapping table should be handled more generically * or through the configuration interfaces perhaps. */ std::map aiqb_paths = { { "ov13858", "/usr/share/libcamera/ipa/ipu3/00ov13858.aiqb" }, { "ov5670", "/usr/share/libcamera/ipa/ipu3/01ov5670.aiqb" }, { "imx258", "/etc/camera/ipu3/00imx258.aiqb" }, }; LOG(IPAIPU3, Info) << "Initialising IPA IPU3 for " << settings.sensorModel; auto it = aiqb_paths.find(settings.sensorModel); if (it == aiqb_paths.end()) { LOG(IPAIPU3, Error) << "Failed to identify tuning data"; return -EINVAL; } LOG(IPAIPU3, Info) << "Using tuning file: " << it->second; ret = aiqb_.load(it->second.c_str()); if (ret) { LOG(IPAIPU3, Error) << "Failed to load AIQB"; return -ENODATA; } /* * Todo: nvm_ and aiqd_ are left as empty nullptrs. * These need to be identified and loaded as required. */ ret = aiq_.init(aiqb_, nvm_, aiqd_); if (ret) return ret; ret = aic_.init(aiqb_); if (ret) return ret; aiqInputParams_.init(); return 0; } int IPAIPU3::start() { setControls(0); return 0; } int IPAIPU3::configure(const IPAConfigInfo &configInfo) { if (configInfo.entityControls.empty()) { LOG(IPAIPU3, Error) << "No controls provided"; return -ENODATA; } sensorInfo_ = configInfo.sensorInfo; ctrls_ = configInfo.entityControls.at(0); const auto itExp = ctrls_.find(V4L2_CID_EXPOSURE); if (itExp == ctrls_.end()) { LOG(IPAIPU3, Error) << "Can't find exposure control"; return -EOWNERDEAD; //EINVAL; 130 } const auto itGain = ctrls_.find(V4L2_CID_ANALOGUE_GAIN); if (itGain == ctrls_.end()) { LOG(IPAIPU3, Error) << "Can't find gain control"; return -ENOKEY; //EINVAL; 126 } minExposure_ = std::max(itExp->second.min().get(), 1); maxExposure_ = itExp->second.max().get(); exposure_ = maxExposure_; minGain_ = std::max(itGain->second.min().get(), 1); maxGain_ = itGain->second.max().get(); gain_ = maxGain_; int ret; ret = aiq_.configure(); if (ret) { LOG(IPAIPU3, Error) << "Failed to configure the AIQ"; return ret; } ret = aiqInputParams_.configure(configInfo); if (ret) { LOG(IPAIPU3, Error) << "Failed to configure AiqInputParams"; return ret; } ret = aic_.configure(configInfo); if (ret) { LOG(IPAIPU3, Error) << "Failed to configure the AIC"; return ret; } /* Set AE/AWB defaults, this typically might not belong here */ aiqInputParams_.setAeAwbAfDefaults(); return 0; } void IPAIPU3::mapBuffers(const std::vector &buffers) { for (const IPABuffer &buffer : buffers) { const FrameBuffer fb(buffer.planes); buffers_.emplace(buffer.id, MappedFrameBuffer(&fb, PROT_READ | PROT_WRITE)); } } void IPAIPU3::unmapBuffers(const std::vector &ids) { for (unsigned int id : ids) { auto it = buffers_.find(id); if (it == buffers_.end()) continue; buffers_.erase(it); } } void IPAIPU3::processEvent(const IPU3Event &event) { switch (event.op) { case EventProcessControls: { processControls(event.frame, event.controls); break; } case EventStatReady: { auto it = buffers_.find(event.bufferId); if (it == buffers_.end()) { LOG(IPAIPU3, Error) << "Could not find stats buffer!"; return; } Span mem = it->second.maps()[0]; const ipu3_uapi_stats_3a *stats = reinterpret_cast(mem.data()); parseStatistics(event.frame, event.frameTimestamp, stats); break; } case EventFillParams: { auto it = buffers_.find(event.bufferId); if (it == buffers_.end()) { LOG(IPAIPU3, Error) << "Could not find param buffer!"; return; } Span mem = it->second.maps()[0]; ipu3_uapi_params *params = reinterpret_cast(mem.data()); fillParams(event.frame, params); break; } default: LOG(IPAIPU3, Error) << "Unknown event " << event.op; break; } } void IPAIPU3::processControls([[maybe_unused]] unsigned int frame, [[maybe_unused]] const ControlList &controls) { /* \todo Start processing for 'frame' based on 'controls'. */ } void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params) { /* Prepare parameters buffer. */ memset(params, 0, sizeof(*params)); /* * Call into the AIQ object, and set up the library with any requested * controls or settings from the incoming request. * * (statistics are fed into the library as a separate event * when available) * * - Run algorithms * * - Fill params buffer with the results of the algorithms. */ /* Run algorithms into/using this context structure */ if (frame % 10 == 0) aiq_.run2a(frame, aiqInputParams_, results_); aic_.updateRuntimeParams(results_); aic_.run(params); exposure_ = results_.ae()->exposures[0].sensor_exposure->coarse_integration_time; gain_ = results_.ae()->exposures[0].sensor_exposure->analog_gain_code_global; setControls(frame); IPU3Action op; op.op = ActionParamFilled; queueFrameAction.emit(frame, op); } void IPAIPU3::parseStatistics(unsigned int frame, int64_t frameTimestamp, const ipu3_uapi_stats_3a *stats) { ControlList ctrls(controls::controls); /* \todo React to statistics and update internal state machine. */ /* *stats comes from the IPU3 hardware. We need to give this data into * the AIQ library. */ ASSERT (frameTimestamp > 0); aiq_.setStatistics(frame, frameTimestamp, results_, stats); /* Set frame durations from exposure results */ ia_aiq_exposure_sensor_parameters *sensorExposure = results_.ae()->exposures->sensor_exposure; int64_t frameDuration = (sensorExposure->line_length_pixels * sensorExposure->frame_length_lines) / (sensorInfo_.pixelRate / 1e6); ctrls.set(controls::FrameDuration, frameDuration); double fps = frameTimestamp - lastFrameTimestamp_; fps = lastFrameTimestamp_ != 0 && fps ? 1000000000.0 / fps : 0.0; lastFrameTimestamp_ = frameTimestamp; LOG(IPAIPU3, Info) << "Frame " << frame << ": fps " << fps; IPU3Action op; op.op = ActionMetadataReady; op.controls = ctrls; queueFrameAction.emit(frame, op); } void IPAIPU3::setControls(unsigned int frame) { IPU3Action op; op.op = ActionSetSensorControls; ControlList ctrls(ctrls_); ctrls.set(V4L2_CID_EXPOSURE, static_cast(exposure_)); ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(gain_)); op.controls = ctrls; queueFrameAction.emit(frame, op); } } /* namespace ipa::ipu3 */ /* * External IPA module interface */ extern "C" { const struct IPAModuleInfo ipaModuleInfo = { IPA_MODULE_API_VERSION, 1, "PipelineHandlerIPU3", "ipu3", }; IPAInterface *ipaCreate() { return new ipa::ipu3::IPAIPU3(); } } } /* namespace libcamera */