diff options
author | Jacopo Mondi <jacopo.mondi@ideasonboard.com> | 2024-10-10 08:25:33 +0200 |
---|---|---|
committer | Jacopo Mondi <jacopo.mondi@ideasonboard.com> | 2024-10-11 12:45:34 +0200 |
commit | 2f533ab71e8ed3c26805e6d0f497ed8dbb56720e (patch) | |
tree | e43497473faca70c3481bfda5f107613ddfd7de1 | |
parent | dc4af4914a13e18285ee1a62baef3f7ac702867c (diff) |
WIP: rework rkisp1 pipeline
Signed-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
-rw-r--r-- | include/libcamera/ipa/rkisp1.mojom | 10 | ||||
-rw-r--r-- | src/ipa/rkisp1/rkisp1.cpp | 12 | ||||
-rw-r--r-- | src/libcamera/pipeline/rkisp1/rkisp1.cpp | 375 |
3 files changed, 113 insertions, 284 deletions
diff --git a/include/libcamera/ipa/rkisp1.mojom b/include/libcamera/ipa/rkisp1.mojom index 80d54a03..ced1526e 100644 --- a/include/libcamera/ipa/rkisp1.mojom +++ b/include/libcamera/ipa/rkisp1.mojom @@ -31,13 +31,13 @@ interface IPARkISP1Interface { unmapBuffers(array<uint32> ids); [async] queueRequest(uint32 frame, libcamera.ControlList reqControls); - [async] fillParamsBuffer(uint32 frame, uint32 bufferId); - [async] processStatsBuffer(uint32 frame, uint32 bufferId, - libcamera.ControlList sensorControls); + [async] computeParams(uint32 frame, uint32 bufferId); + [async] processStats(uint32 frame, uint32 bufferId, + libcamera.ControlList sensorControls); }; interface IPARkISP1EventInterface { - paramsBufferReady(uint32 frame, uint32 bytesused); + paramsComputed(uint32 frame, uint32 bytesused); setSensorControls(uint32 frame, libcamera.ControlList sensorControls); - metadataReady(uint32 frame, libcamera.ControlList metadata); + statsProcessed(uint32 frame, libcamera.ControlList metadata); }; diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index 9e161cab..005c17ca 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -65,8 +65,8 @@ public: void unmapBuffers(const std::vector<unsigned int> &ids) override; void queueRequest(const uint32_t frame, const ControlList &controls) override; - void fillParamsBuffer(const uint32_t frame, const uint32_t bufferId) override; - void processStatsBuffer(const uint32_t frame, const uint32_t bufferId, + void computeParams(const uint32_t frame, const uint32_t bufferId) override; + void processStats(const uint32_t frame, const uint32_t bufferId, const ControlList &sensorControls) override; protected: @@ -335,7 +335,7 @@ void IPARkISP1::queueRequest(const uint32_t frame, const ControlList &controls) } } -void IPARkISP1::fillParamsBuffer(const uint32_t frame, const uint32_t bufferId) +void IPARkISP1::computeParams(const uint32_t frame, const uint32_t bufferId) { IPAFrameContext &frameContext = context_.frameContexts.get(frame); @@ -345,10 +345,10 @@ void IPARkISP1::fillParamsBuffer(const uint32_t frame, const uint32_t bufferId) for (auto const &algo : algorithms()) algo->prepare(context_, frame, frameContext, ¶ms); - paramsBufferReady.emit(frame, params.size()); + paramsComputed.emit(frame, params.size()); } -void IPARkISP1::processStatsBuffer(const uint32_t frame, const uint32_t bufferId, +void IPARkISP1::processStats(const uint32_t frame, const uint32_t bufferId, const ControlList &sensorControls) { IPAFrameContext &frameContext = context_.frameContexts.get(frame); @@ -378,7 +378,7 @@ void IPARkISP1::processStatsBuffer(const uint32_t frame, const uint32_t bufferId setControls(frame); - metadataReady.emit(frame, metadata); + statsProcessed.emit(bufferId, metadata); } void IPARkISP1::updateControls(const IPACameraSensorInfo &sensorInfo, diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp index e0774328..af30584e 100644 --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp @@ -9,7 +9,7 @@ #include <memory> #include <numeric> #include <optional> -#include <queue> +#include <deque> #include <linux/media-bus-format.h> #include <linux/rkisp1-config.h> @@ -50,33 +50,15 @@ LOG_DEFINE_CATEGORY(RkISP1) class PipelineHandlerRkISP1; class RkISP1CameraData; -struct RkISP1FrameInfo { - unsigned int frame; - - FrameBuffer *paramBuffer; - FrameBuffer *statBuffer; - FrameBuffer *mainPathBuffer; - FrameBuffer *selfPathBuffer; - - bool paramDequeued; - bool metadataProcessed; -}; - -class RkISP1Frames +class RkISP1Capture { public: - RkISP1Frames(PipelineHandler *pipe); - - RkISP1FrameInfo *create(const RkISP1CameraData *data, bool isRaw); - int destroy(unsigned int frame); - void clear(); - - RkISP1FrameInfo *find(unsigned int frame); - RkISP1FrameInfo *find(FrameBuffer *buffer); + RkISP1Capture(FrameBuffer *paramBuffer) + : paramBuffer_(paramBuffer) + { + } -private: - PipelineHandlerRkISP1 *pipe_; - std::map<unsigned int, RkISP1FrameInfo *> frameInfo_; + FrameBuffer *paramBuffer_; }; class RkISP1CameraData : public Camera::Private @@ -84,7 +66,7 @@ class RkISP1CameraData : public Camera::Private public: RkISP1CameraData(PipelineHandler *pipe, RkISP1MainPath *mainPath, RkISP1SelfPath *selfPath) - : Camera::Private(pipe), frame_(0), frameInfo_(pipe), + : Camera::Private(pipe), frame_(0), mainPath_(mainPath), selfPath_(selfPath) { } @@ -98,19 +80,20 @@ public: std::unique_ptr<DelayedControls> delayedCtrls_; unsigned int frame_; std::vector<IPABuffer> ipaBuffers_; - RkISP1Frames frameInfo_; RkISP1MainPath *mainPath_; RkISP1SelfPath *selfPath_; std::unique_ptr<ipa::rkisp1::IPAProxyRkISP1> ipa_; + std::map<unsigned int, RkISP1Capture> captures_; + private: - void paramFilled(unsigned int frame, unsigned int bytesused); + void paramsComputed(unsigned int frame, unsigned int bytesused); void setSensorControls(unsigned int frame, const ControlList &sensorControls); - void metadataReady(unsigned int frame, const ControlList &metadata); + void statsProcessed(unsigned int frame, const ControlList &metadata); }; class RkISP1CameraConfiguration : public CameraConfiguration @@ -166,15 +149,13 @@ private: } friend RkISP1CameraData; - friend RkISP1Frames; int initLinks(Camera *camera, const CameraSensor *sensor, const RkISP1CameraConfiguration &config); int createCamera(MediaEntity *sensor); - void tryCompleteRequest(RkISP1FrameInfo *info); void bufferReady(FrameBuffer *buffer); - void paramReady(FrameBuffer *buffer); - void statReady(FrameBuffer *buffer); + void paramBufferReady(FrameBuffer *buffer); + void statBufferReady(FrameBuffer *buffer); void frameStart(uint32_t sequence); int allocateBuffers(Camera *camera); @@ -194,134 +175,19 @@ private: std::vector<std::unique_ptr<FrameBuffer>> paramBuffers_; std::vector<std::unique_ptr<FrameBuffer>> statBuffers_; - std::queue<FrameBuffer *> availableParamBuffers_; - std::queue<FrameBuffer *> availableStatBuffers_; + std::deque<FrameBuffer *> availableParamBuffers_; + std::deque<FrameBuffer *> availableStatBuffers_; std::vector<std::unique_ptr<FrameBuffer>> mainPathBuffers_; std::vector<std::unique_ptr<FrameBuffer>> selfPathBuffers_; - std::queue<FrameBuffer *> availableMainPathBuffers_; - std::queue<FrameBuffer *> availableSelfPathBuffers_; + std::deque<FrameBuffer *> availableMainPathBuffers_; + std::deque<FrameBuffer *> availableSelfPathBuffers_; Camera *activeCamera_; const MediaPad *ispSink_; }; -RkISP1Frames::RkISP1Frames(PipelineHandler *pipe) - : pipe_(static_cast<PipelineHandlerRkISP1 *>(pipe)) -{ -} - -RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, bool isRaw) -{ - unsigned int frame = data->frame_; - - FrameBuffer *paramBuffer = nullptr; - FrameBuffer *statBuffer = nullptr; - FrameBuffer *mainPathBuffer = nullptr; - - if (!isRaw) { - if (pipe_->availableParamBuffers_.empty()) { - LOG(RkISP1, Error) << "Parameters buffer underrun"; - return nullptr; - } - - if (pipe_->availableStatBuffers_.empty()) { - LOG(RkISP1, Error) << "Statistic buffer underrun"; - return nullptr; - } - - paramBuffer = pipe_->availableParamBuffers_.front(); - pipe_->availableParamBuffers_.pop(); - - statBuffer = pipe_->availableStatBuffers_.front(); - pipe_->availableStatBuffers_.pop(); - } - - if (pipe_->availableMainPathBuffers_.empty()) { - LOG(RkISP1, Error) << "Main path buffer underrun"; - return nullptr; - } - mainPathBuffer = pipe_->availableMainPathBuffers_.front(); - pipe_->availableMainPathBuffers_.pop(); - - //FrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_); - //FrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_); - - RkISP1FrameInfo *info = new RkISP1FrameInfo; - - info->frame = frame; - info->paramBuffer = paramBuffer; - info->mainPathBuffer = mainPathBuffer; - //info->selfPathBuffer = selfPathBuffer; - info->statBuffer = statBuffer; - info->paramDequeued = false; - info->metadataProcessed = false; - - frameInfo_[frame] = info; - - return info; -} - -int RkISP1Frames::destroy(unsigned int frame) -{ - RkISP1FrameInfo *info = find(frame); - if (!info) - return -ENOENT; - - pipe_->availableParamBuffers_.push(info->paramBuffer); - pipe_->availableStatBuffers_.push(info->statBuffer); - - frameInfo_.erase(info->frame); - - delete info; - - return 0; -} - -void RkISP1Frames::clear() -{ - for (const auto &entry : frameInfo_) { - RkISP1FrameInfo *info = entry.second; - - pipe_->availableParamBuffers_.push(info->paramBuffer); - pipe_->availableStatBuffers_.push(info->statBuffer); - - delete info; - } - - frameInfo_.clear(); -} - -RkISP1FrameInfo *RkISP1Frames::find(unsigned int frame) -{ - auto itInfo = frameInfo_.find(frame); - - if (itInfo != frameInfo_.end()) - return itInfo->second; - - LOG(RkISP1, Fatal) << "Can't locate info from frame"; - - return nullptr; -} - -RkISP1FrameInfo *RkISP1Frames::find(FrameBuffer *buffer) -{ - for (auto &itInfo : frameInfo_) { - RkISP1FrameInfo *info = itInfo.second; - - if (info->paramBuffer == buffer || - info->statBuffer == buffer || - info->mainPathBuffer == buffer || - info->selfPathBuffer == buffer) - return info; - } - - LOG(RkISP1, Fatal) << "Can't locate info from buffer"; - - return nullptr; -} - PipelineHandlerRkISP1 *RkISP1CameraData::pipe() { return static_cast<PipelineHandlerRkISP1 *>(Camera::Private::pipe()); @@ -334,8 +200,8 @@ int RkISP1CameraData::loadIPA(unsigned int hwRevision) return -ENOENT; ipa_->setSensorControls.connect(this, &RkISP1CameraData::setSensorControls); - ipa_->paramsBufferReady.connect(this, &RkISP1CameraData::paramFilled); - ipa_->metadataReady.connect(this, &RkISP1CameraData::metadataReady); + ipa_->paramsComputed.connect(this, &RkISP1CameraData::paramsComputed); + ipa_->statsProcessed.connect(this, &RkISP1CameraData::statsProcessed); /* * The API tuning file is made from the sensor name unless the @@ -367,22 +233,32 @@ int RkISP1CameraData::loadIPA(unsigned int hwRevision) return 0; } -void RkISP1CameraData::paramFilled(unsigned int frame, unsigned int bytesused) +void RkISP1CameraData::paramsComputed(unsigned int frame, unsigned int bytesused) { PipelineHandlerRkISP1 *pipe = RkISP1CameraData::pipe(); - RkISP1FrameInfo *info = frameInfo_.find(frame); - if (!info) - return; - info->paramBuffer->_d()->metadata().planes()[0].bytesused = bytesused; - pipe->param_->queueBuffer(info->paramBuffer); - pipe->stat_->queueBuffer(info->statBuffer); + auto it = captures_.find(frame); + ASSERT(it != captures_.end()); + + RkISP1Capture &capture = it->second; - if (info->mainPathBuffer) - mainPath_->queueBuffer(info->mainPathBuffer); + FrameBuffer *paramBuffer = capture.paramBuffer_; + ASSERT(paramBuffer); - if (selfPath_ && info->selfPathBuffer) - selfPath_->queueBuffer(info->selfPathBuffer); + paramBuffer->_d()->metadata().planes()[0].bytesused = bytesused; + pipe->param_->queueBuffer(paramBuffer); + + captures_.erase(it); + + /* Queue main path video buffer. */ + if (pipe->availableMainPathBuffers_.empty()) { + LOG(RkISP1, Error) << "Main path buffer underrun"; + return; + } + FrameBuffer *mainPathBuffer = pipe->availableMainPathBuffers_.front(); + pipe->availableMainPathBuffers_.pop_front(); + + pipe->mainPath_.queueBuffer(mainPathBuffer); } void RkISP1CameraData::setSensorControls([[maybe_unused]] unsigned int frame, @@ -391,17 +267,36 @@ void RkISP1CameraData::setSensorControls([[maybe_unused]] unsigned int frame, delayedCtrls_->push(sensorControls); } -void RkISP1CameraData::metadataReady(unsigned int frame, +void RkISP1CameraData::statsProcessed(unsigned int bufferId, [[maybe_unused]] const ControlList &metadata) { - RkISP1FrameInfo *info = frameInfo_.find(frame); - if (!info) - return; + PipelineHandlerRkISP1 *pipe = RkISP1CameraData::pipe(); - //info->request->metadata().merge(metadata); - info->metadataProcessed = true; + /* Find the statistics buffer that has been processed by the IPA. */ + FrameBuffer *statBuffer = nullptr; + for (auto buffer : pipe->availableStatBuffers_) { + if (buffer->cookie() == bufferId) { + statBuffer = buffer; + break; + } + } + + ASSERT(statBuffer); + + /* Queue the buffer back to the stats video device. */ + unsigned int frameNumber = statBuffer->metadata().sequence; + pipe->stat_->queueBuffer(statBuffer); + + /* Request the IPA to produce parameters. */ + if (pipe->availableParamBuffers_.empty()) { + LOG(RkISP1, Error) << "Parameters buffer underrun"; + return; + } + FrameBuffer *paramBuffer = pipe->availableParamBuffers_.front(); + pipe->availableParamBuffers_.pop_front(); - pipe()->tryCompleteRequest(info); + captures_.emplace(frameNumber, paramBuffer); + ipa_->computeParams(frameNumber, paramBuffer->cookie()); } /* ----------------------------------------------------------------------------- @@ -884,14 +779,14 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) buffer->setCookie(ipaBufferId++); data->ipaBuffers_.emplace_back(buffer->cookie(), buffer->planes()); - availableParamBuffers_.push(buffer.get()); + availableParamBuffers_.push_back(buffer.get()); } for (std::unique_ptr<FrameBuffer> &buffer : statBuffers_) { buffer->setCookie(ipaBufferId++); data->ipaBuffers_.emplace_back(buffer->cookie(), buffer->planes()); - availableStatBuffers_.push(buffer.get()); + availableStatBuffers_.push_back(buffer.get()); } data->ipa_->mapBuffers(data->ipaBuffers_); @@ -902,7 +797,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) goto error; for (std::unique_ptr<FrameBuffer> &buffer : mainPathBuffers_) - availableMainPathBuffers_.push(buffer.get()); + availableMainPathBuffers_.push_back(buffer.get()); if (hasSelfPath_) { ret = selfPath_.exportBuffers(maxCount, &selfPathBuffers_); @@ -910,7 +805,7 @@ int PipelineHandlerRkISP1::allocateBuffers(Camera *camera) goto error; for (std::unique_ptr<FrameBuffer> &buffer : selfPathBuffers_) - availableSelfPathBuffers_.push(buffer.get()); + availableSelfPathBuffers_.push_back(buffer.get()); } return 0; @@ -929,10 +824,10 @@ int PipelineHandlerRkISP1::freeBuffers(Camera *camera) RkISP1CameraData *data = cameraData(camera); while (!availableStatBuffers_.empty()) - availableStatBuffers_.pop(); + availableStatBuffers_.pop_front(); while (!availableParamBuffers_.empty()) - availableParamBuffers_.pop(); + availableParamBuffers_.pop_front(); paramBuffers_.clear(); statBuffers_.clear(); @@ -965,6 +860,15 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL return ret; actions += [&]() { freeBuffers(camera); }; + /* Pre-queue statistics buffers to the stat video device. */ + for (auto statBuf : availableStatBuffers_) { + ret = stat_->queueBuffer(statBuf); + if (ret) { + LOG(RkISP1, Error) << "Failed to pre-queue stat buffers"; + return ret; + } + } + ret = data->ipa_->start(); if (ret) { LOG(RkISP1, Error) @@ -994,10 +898,15 @@ int PipelineHandlerRkISP1::start(Camera *camera, [[maybe_unused]] const ControlL } if (data->mainPath_->isEnabled()) { + ret = mainPath_.preQueueBuffers(availableMainPathBuffers_); + if (ret) + return ret; + ret = mainPath_.start(); if (ret) return ret; actions += [&]() { mainPath_.stop(); }; + } if (hasSelfPath_ && data->selfPath_->isEnabled()) { @@ -1040,35 +949,15 @@ void PipelineHandlerRkISP1::stopDevice(Camera *camera) } ASSERT(data->queuedRequests_.empty()); - data->frameInfo_.clear(); freeBuffers(camera); activeCamera_ = nullptr; } -int PipelineHandlerRkISP1::queueRequestDevice(Camera *camera, Request *request) +int PipelineHandlerRkISP1::queueRequestDevice([[maybe_unused]] Camera *camera, + [[maybe_unused]] Request *request) { - RkISP1CameraData *data = cameraData(camera); - - RkISP1FrameInfo *info = data->frameInfo_.create(data, isRaw_); - if (!info) - return -ENOENT; - - data->ipa_->queueRequest(data->frame_, request->controls()); - if (isRaw_) { - if (info->mainPathBuffer) - data->mainPath_->queueBuffer(info->mainPathBuffer); - - if (data->selfPath_ && info->selfPathBuffer) - data->selfPath_->queueBuffer(info->selfPathBuffer); - } else { - data->ipa_->fillParamsBuffer(data->frame_, - info->paramBuffer->cookie()); - } - - data->frame_++; - return 0; } @@ -1239,8 +1128,8 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) mainPath_.bufferReady().connect(this, &PipelineHandlerRkISP1::bufferReady); if (hasSelfPath_) selfPath_.bufferReady().connect(this, &PipelineHandlerRkISP1::bufferReady); - stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady); - param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady); + stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statBufferReady); + param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramBufferReady); /* * Enumerate all sensors connected to the ISP and create one @@ -1259,94 +1148,34 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator) * Buffer Handling */ -void PipelineHandlerRkISP1::tryCompleteRequest(RkISP1FrameInfo *info) -{ - RkISP1CameraData *data = cameraData(activeCamera_); - //Request *request = info->request; - - //if (request->hasPendingBuffers()) - return; - - if (!info->metadataProcessed) - return; - - if (!isRaw_ && !info->paramDequeued) - return; - - data->frameInfo_.destroy(info->frame); - - //completeRequest(request); -} - void PipelineHandlerRkISP1::bufferReady(FrameBuffer *buffer) { ASSERT(activeCamera_); - RkISP1CameraData *data = cameraData(activeCamera_); - - RkISP1FrameInfo *info = data->frameInfo_.find(buffer); - if (!info) - return; - const FrameMetadata &metadata = buffer->metadata(); - Request *request = buffer->request(); + unsigned int frameNumber = buffer->metadata().sequence; + LOG(Error) << "GOT A FRAME! " << frameNumber; - if (metadata.status != FrameMetadata::FrameCancelled) { - /* - * Record the sensor's timestamp in the request metadata. - * - * \todo The sensor timestamp should be better estimated by connecting - * to the V4L2Device::frameStart signal. - */ - request->metadata().set(controls::SensorTimestamp, - metadata.timestamp); - - if (isRaw_) { - const ControlList &ctrls = - data->delayedCtrls_->get(metadata.sequence); - data->ipa_->processStatsBuffer(info->frame, 0, ctrls); - } - } else { - if (isRaw_) - info->metadataProcessed = true; - } - - completeBuffer(request, buffer); - tryCompleteRequest(info); + availableMainPathBuffers_.push_back(buffer); } -void PipelineHandlerRkISP1::paramReady(FrameBuffer *buffer) +void PipelineHandlerRkISP1::paramBufferReady(FrameBuffer *paramBuffer) { ASSERT(activeCamera_); - RkISP1CameraData *data = cameraData(activeCamera_); - - RkISP1FrameInfo *info = data->frameInfo_.find(buffer); - if (!info) - return; - info->paramDequeued = true; - tryCompleteRequest(info); + /* Requeue back the buffer to the params node. */ + availableParamBuffers_.push_back(paramBuffer); } -void PipelineHandlerRkISP1::statReady(FrameBuffer *buffer) +void PipelineHandlerRkISP1::statBufferReady(FrameBuffer *statBuffer) { ASSERT(activeCamera_); RkISP1CameraData *data = cameraData(activeCamera_); - RkISP1FrameInfo *info = data->frameInfo_.find(buffer); - if (!info) - return; - - if (buffer->metadata().status == FrameMetadata::FrameCancelled) { - info->metadataProcessed = true; - tryCompleteRequest(info); - return; - } - - if (data->frame_ <= buffer->metadata().sequence) - data->frame_ = buffer->metadata().sequence + 1; + unsigned int frameNumber = statBuffer->metadata().sequence; + LOG(Error) << "Stats buffer: " << frameNumber; - data->ipa_->processStatsBuffer(info->frame, info->statBuffer->cookie(), - data->delayedCtrls_->get(buffer->metadata().sequence)); + data->ipa_->processStats(frameNumber, statBuffer->cookie(), + data->delayedCtrls_->get(frameNumber)); } REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1, "rkisp1") |