From b612496fc4f96677cde61d6a65247335ea0185c2 Mon Sep 17 00:00:00 2001 From: Umang Jain Date: Thu, 21 Jul 2022 13:13:02 +0100 Subject: ipa: libipa: Introduce FrameContextQueue Introduce a common implementation in libipa to represent the queue of frame contexts. Signed-off-by: Umang Jain Signed-off-by: Kieran Bingham Signed-off-by: Jacopo Mondi Reviewed-by: Kieran Bingham Signed-off-by: Laurent Pinchart --- src/ipa/libipa/fc_queue.cpp | 118 ++++++++++++++++++++++++++++++++++++++++++++ src/ipa/libipa/fc_queue.h | 108 ++++++++++++++++++++++++++++++++++++++++ src/ipa/libipa/meson.build | 2 + 3 files changed, 228 insertions(+) create mode 100644 src/ipa/libipa/fc_queue.cpp create mode 100644 src/ipa/libipa/fc_queue.h diff --git a/src/ipa/libipa/fc_queue.cpp b/src/ipa/libipa/fc_queue.cpp new file mode 100644 index 00000000..57a36951 --- /dev/null +++ b/src/ipa/libipa/fc_queue.cpp @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2022, Google Inc. + * + * fc_queue.cpp - IPA Frame context queue + */ + +#include "fc_queue.h" + +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(FCQueue) + +namespace ipa { + +/** + * \file fc_queue.h + * \brief Queue of per-frame contexts + */ + +/** + * \class FCQueue + * \brief A support class for managing FrameContext instances in IPA modules + * \tparam FrameContext The IPA module-specific FrameContext derived class type + * + * Along with the Module and Algorithm classes, the frame context queue is a + * core component of the libipa infrastructure. It stores per-frame contexts + * used by the Algorithm operations. By centralizing the lifetime management of + * the contexts and implementing safeguards against underflows and overflows, it + * simplifies IPA modules and improves their reliability. + * + * The queue references frame contexts by a monotonically increasing sequence + * number. The FCQueue design assumes that this number matches both the sequence + * number of the corresponding frame, as generated by the camera sensor, and the + * sequence number of the request. This allows IPA modules to obtain the frame + * context from any location where a request or a frame is available. + * + * A frame context normally begins its lifetime when the corresponding request + * is queued, way before the frame is captured by the camera sensor. IPA modules + * allocate the context from the queue at that point, calling alloc() using the + * request number. The queue initializes the context, and the IPA module then + * populates it with data from the request. The context can be later retrieved + * with a call to get(), typically when the IPA module is requested to provide + * sensor or ISP parameters or receives statistics for a frame. The frame number + * is used at that point to identify the context. + * + * If an application fails to queue requests to the camera fast enough, frames + * may be produced by the camera sensor and processed by the IPA module without + * a corresponding request having been queued to the IPA module. This creates an + * underrun condition, where the IPA module will try to get a frame context that + * hasn't been allocated. In this case, the get() function will allocate and + * initialize a context for the frame, and log a message. Algorithms will not + * apply the controls associated with the late request, but should otherwise + * behave correctly. + * + * \todo Mark the frame context with a per-frame control error flag in case of + * underrun, and research how algorithms should handle this. + * + * At its core, the queue uses a circular buffer to avoid dynamic memory + * allocation at runtime. The buffer is pre-allocated with a maximum number of + * entries when the FCQueue instance is constructed. Entries are initialized on + * first use by alloc() or, in underrun conditions, get(). The queue is not + * allowed to overflow, which must be ensured by pipeline handlers never + * queuing more in-flight requests to the IPA module than the queue size. If an + * overflow condition is detected, the queue will log a fatal error. + */ + +/** + * \fn FCQueue::FCQueue(unsigned int size) + * \brief Construct a frame contexts queue of a specified size + * \param[in] size The number of contexts in the queue + */ + +/** + * \fn FCQueue::clear() + * \brief Clear the contexts queue + * + * IPA modules must clear the frame context queue at the beginning of a new + * streaming session, in IPAModule::start(). + * + * \todo Fix any issue this may cause with requests queued before the camera is + * started. + */ + +/** + * \fn FCQueue::alloc(uint32_t frame) + * \brief Allocate and return a FrameContext for the \a frame + * \param[in] frame The frame context sequence number + * + * The first call to obtain a FrameContext from the FCQueue should be handled + * through this function. The FrameContext will be initialised, if not + * initialised already, and returned to the caller. + * + * If the FrameContext was already initialized for this \a frame, a warning will + * be reported and the previously initialized FrameContext is returned. + * + * Frame contexts are expected to be initialised when a Request is first passed + * to the IPA module in IPAModule::queueRequest(). + * + * \return A reference to the FrameContext for sequence \a frame + */ + +/** + * \fn FCQueue::get(uint32_t frame) + * \brief Obtain the FrameContext for the \a frame + * \param[in] frame The frame context sequence number + * + * If the FrameContext is not correctly initialised for the \a frame, it will be + * initialised. + * + * \return A reference to the FrameContext for sequence \a frame + */ + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/fc_queue.h b/src/ipa/libipa/fc_queue.h new file mode 100644 index 00000000..4f5cb5d3 --- /dev/null +++ b/src/ipa/libipa/fc_queue.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2022, Google Inc. + * + * fc_queue.h - IPA Frame context queue + */ + +#pragma once + +#include + +#include + +namespace libcamera { + +LOG_DECLARE_CATEGORY(FCQueue) + +namespace ipa { + +template +class FCQueue +{ +public: + FCQueue(unsigned int size) + : contexts_(size) + { + } + + void clear() + { + for (FrameContext &ctx : contexts_) + ctx.frame = 0; + } + + FrameContext &alloc(const uint32_t frame) + { + FrameContext &frameContext = contexts_[frame % contexts_.size()]; + + /* + * Do not re-initialise if a get() call has already fetched this + * frame context to preseve the context. + * + * \todo If the the sequence number of the context to initialise + * is smaller than the sequence number of the queue slot to use, + * it means that we had a serious request underrun and more + * frames than the queue size has been produced since the last + * time the application has queued a request. Does this deserve + * an error condition ? + */ + if (frame != 0 && frame <= frameContext.frame) + LOG(FCQueue, Warning) + << "Frame " << frame << " already initialised"; + else + init(frameContext, frame); + + return frameContext; + } + + FrameContext &get(uint32_t frame) + { + FrameContext &frameContext = contexts_[frame % contexts_.size()]; + + /* + * If the IPA algorithms try to access a frame context slot which + * has been already overwritten by a newer context, it means the + * frame context queue has overflowed and the desired context + * has been forever lost. The pipeline handler shall avoid + * queueing more requests to the IPA than the frame context + * queue size. + */ + if (frame < frameContext.frame) + LOG(FCQueue, Fatal) << "Frame context for " << frame + << " has been overwritten by " + << frameContext.frame; + + if (frame == frameContext.frame) + return frameContext; + + /* + * The frame context has been retrieved before it was + * initialised through the initialise() call. This indicates an + * algorithm attempted to access a Frame context before it was + * queued to the IPA. Controls applied for this request may be + * left unhandled. + * + * \todo Set an error flag for per-frame control errors. + */ + LOG(FCQueue, Warning) + << "Obtained an uninitialised FrameContext for " << frame; + + init(frameContext, frame); + + return frameContext; + } + +private: + void init(FrameContext &frameContext, const uint32_t frame) + { + frameContext = {}; + frameContext.frame = frame; + } + + std::vector contexts_; +}; + +} /* namespace ipa */ + +} /* namespace libcamera */ diff --git a/src/ipa/libipa/meson.build b/src/ipa/libipa/meson.build index fb894bc6..016b8e0e 100644 --- a/src/ipa/libipa/meson.build +++ b/src/ipa/libipa/meson.build @@ -3,6 +3,7 @@ libipa_headers = files([ 'algorithm.h', 'camera_sensor_helper.h', + 'fc_queue.h', 'histogram.h', 'module.h', ]) @@ -10,6 +11,7 @@ libipa_headers = files([ libipa_sources = files([ 'algorithm.cpp', 'camera_sensor_helper.cpp', + 'fc_queue.cpp', 'histogram.cpp', 'module.cpp', ]) -- cgit v1.2.1