summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libcamera/pipeline/rkisp1/meson.build1
-rw-r--r--src/libcamera/pipeline/rkisp1/rkisp1.cpp548
-rw-r--r--src/libcamera/pipeline/rkisp1/timeline.cpp227
-rw-r--r--src/libcamera/pipeline/rkisp1/timeline.h72
4 files changed, 831 insertions, 17 deletions
diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build
index f1cc4046..d04fb452 100644
--- a/src/libcamera/pipeline/rkisp1/meson.build
+++ b/src/libcamera/pipeline/rkisp1/meson.build
@@ -1,3 +1,4 @@
libcamera_sources += files([
'rkisp1.cpp',
+ 'timeline.cpp',
])
diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
index de4ab523..029d5868 100644
--- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp
+++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
@@ -9,32 +9,115 @@
#include <array>
#include <iomanip>
#include <memory>
-#include <vector>
+#include <queue>
#include <linux/media-bus-format.h>
+#include <ipa/rkisp1.h>
+#include <libcamera/buffer.h>
#include <libcamera/camera.h>
+#include <libcamera/control_ids.h>
#include <libcamera/request.h>
#include <libcamera/stream.h>
#include "camera_sensor.h"
#include "device_enumerator.h"
+#include "ipa_manager.h"
#include "log.h"
#include "media_device.h"
#include "pipeline_handler.h"
+#include "timeline.h"
#include "utils.h"
#include "v4l2_subdevice.h"
#include "v4l2_videodevice.h"
+#define RKISP1_PARAM_BASE 0x100
+#define RKISP1_STAT_BASE 0x200
+
namespace libcamera {
LOG_DEFINE_CATEGORY(RkISP1)
+class PipelineHandlerRkISP1;
+class RkISP1ActionQueueBuffers;
+
+enum RkISP1ActionType {
+ SetSensor,
+ SOE,
+ QueueBuffers,
+};
+
+struct RkISP1FrameInfo {
+ unsigned int frame;
+ Request *request;
+
+ Buffer *paramBuffer;
+ Buffer *statBuffer;
+ Buffer *videoBuffer;
+
+ bool paramFilled;
+ bool paramDequeued;
+ bool metadataProcessed;
+};
+
+class RkISP1Frames
+{
+public:
+ RkISP1Frames(PipelineHandler *pipe);
+
+ RkISP1FrameInfo *create(unsigned int frame, Request *request, Stream *stream);
+ int destroy(unsigned int frame);
+
+ RkISP1FrameInfo *find(unsigned int frame);
+ RkISP1FrameInfo *find(Buffer *buffer);
+ RkISP1FrameInfo *find(Request *request);
+
+private:
+ PipelineHandlerRkISP1 *pipe_;
+ std::map<unsigned int, RkISP1FrameInfo *> frameInfo_;
+};
+
+class RkISP1Timeline : public Timeline
+{
+public:
+ RkISP1Timeline()
+ : Timeline()
+ {
+ setDelay(SetSensor, -1, 5);
+ setDelay(SOE, 0, -1);
+ setDelay(QueueBuffers, -1, 10);
+ }
+
+ void bufferReady(Buffer *buffer)
+ {
+ /*
+ * Calculate SOE by taking the end of DMA set by the kernel and applying
+ * the time offsets provideprovided by the IPA to find the best estimate
+ * of SOE.
+ */
+
+ ASSERT(frameOffset(SOE) == 0);
+
+ utils::time_point soe = std::chrono::time_point<utils::clock>()
+ + std::chrono::nanoseconds(buffer->timestamp())
+ + timeOffset(SOE);
+
+ notifyStartOfExposure(buffer->sequence(), soe);
+ }
+
+ void setDelay(unsigned int type, int frame, int msdelay)
+ {
+ utils::duration delay = std::chrono::milliseconds(msdelay);
+ setRawDelay(type, frame, delay);
+ }
+};
+
class RkISP1CameraData : public CameraData
{
public:
RkISP1CameraData(PipelineHandler *pipe)
- : CameraData(pipe), sensor_(nullptr)
+ : CameraData(pipe), sensor_(nullptr), frame_(0),
+ frameInfo_(pipe)
{
}
@@ -43,8 +126,20 @@ public:
delete sensor_;
}
+ int loadIPA();
+
Stream stream_;
CameraSensor *sensor_;
+ unsigned int frame_;
+ std::vector<IPABuffer> ipaBuffers_;
+ RkISP1Frames frameInfo_;
+ RkISP1Timeline timeline_;
+
+private:
+ void queueFrameAction(unsigned int frame,
+ const IPAOperationData &action);
+
+ void metadataReady(unsigned int frame, const ControlList &metadata);
};
class RkISP1CameraConfiguration : public CameraConfiguration
@@ -99,18 +194,235 @@ private:
PipelineHandler::cameraData(camera));
}
+ friend RkISP1ActionQueueBuffers;
+ friend RkISP1CameraData;
+ friend RkISP1Frames;
+
int initLinks();
int createCamera(MediaEntity *sensor);
+ void tryCompleteRequest(Request *request);
void bufferReady(Buffer *buffer);
+ void paramReady(Buffer *buffer);
+ void statReady(Buffer *buffer);
MediaDevice *media_;
V4L2Subdevice *dphy_;
V4L2Subdevice *isp_;
V4L2VideoDevice *video_;
+ V4L2VideoDevice *param_;
+ V4L2VideoDevice *stat_;
+
+ BufferPool paramPool_;
+ BufferPool statPool_;
+
+ std::queue<Buffer *> paramBuffers_;
+ std::queue<Buffer *> statBuffers_;
Camera *activeCamera_;
};
+RkISP1Frames::RkISP1Frames(PipelineHandler *pipe)
+ : pipe_(dynamic_cast<PipelineHandlerRkISP1 *>(pipe))
+{
+}
+
+RkISP1FrameInfo *RkISP1Frames::create(unsigned int frame, Request *request, Stream *stream)
+{
+ if (pipe_->paramBuffers_.empty()) {
+ LOG(RkISP1, Error) << "Parameters buffer underrun";
+ return nullptr;
+ }
+ Buffer *paramBuffer = pipe_->paramBuffers_.front();
+
+ if (pipe_->statBuffers_.empty()) {
+ LOG(RkISP1, Error) << "Statisitc buffer underrun";
+ return nullptr;
+ }
+ Buffer *statBuffer = pipe_->statBuffers_.front();
+
+ Buffer *videoBuffer = request->findBuffer(stream);
+ if (!videoBuffer) {
+ LOG(RkISP1, Error)
+ << "Attempt to queue request with invalid stream";
+ return nullptr;
+ }
+
+ pipe_->paramBuffers_.pop();
+ pipe_->statBuffers_.pop();
+
+ RkISP1FrameInfo *info = new RkISP1FrameInfo;
+
+ info->frame = frame;
+ info->request = request;
+ info->paramBuffer = paramBuffer;
+ info->videoBuffer = videoBuffer;
+ info->statBuffer = statBuffer;
+ info->paramFilled = false;
+ info->paramDequeued = false;
+ info->metadataProcessed = false;
+
+ frameInfo_[frame] = info;
+
+ return info;
+}
+
+int RkISP1Frames::destroy(unsigned int frame)
+{
+ RkISP1FrameInfo *info = find(frame);
+ if (!info)
+ return -ENOENT;
+
+ pipe_->paramBuffers_.push(info->paramBuffer);
+ pipe_->statBuffers_.push(info->statBuffer);
+
+ frameInfo_.erase(info->frame);
+
+ delete info;
+
+ return 0;
+}
+
+RkISP1FrameInfo *RkISP1Frames::find(unsigned int frame)
+{
+ auto itInfo = frameInfo_.find(frame);
+
+ if (itInfo != frameInfo_.end())
+ return itInfo->second;
+
+ LOG(RkISP1, Error) << "Can't locate info from frame";
+ return nullptr;
+}
+
+RkISP1FrameInfo *RkISP1Frames::find(Buffer *buffer)
+{
+ for (auto &itInfo : frameInfo_) {
+ RkISP1FrameInfo *info = itInfo.second;
+
+ if (info->paramBuffer == buffer ||
+ info->statBuffer == buffer ||
+ info->videoBuffer == buffer)
+ return info;
+ }
+
+ LOG(RkISP1, Error) << "Can't locate info from buffer";
+ return nullptr;
+}
+
+RkISP1FrameInfo *RkISP1Frames::find(Request *request)
+{
+ for (auto &itInfo : frameInfo_) {
+ RkISP1FrameInfo *info = itInfo.second;
+
+ if (info->request == request)
+ return info;
+ }
+
+ LOG(RkISP1, Error) << "Can't locate info from request";
+ return nullptr;
+}
+
+class RkISP1ActionSetSensor : public FrameAction
+{
+public:
+ RkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls)
+ : FrameAction(frame, SetSensor), sensor_(sensor), controls_(controls) {}
+
+protected:
+ void run() override
+ {
+ sensor_->setControls(&controls_);
+ }
+
+private:
+ CameraSensor *sensor_;
+ V4L2ControlList controls_;
+};
+
+class RkISP1ActionQueueBuffers : public FrameAction
+{
+public:
+ RkISP1ActionQueueBuffers(unsigned int frame, RkISP1CameraData *data,
+ PipelineHandlerRkISP1 *pipe)
+ : FrameAction(frame, QueueBuffers), data_(data), pipe_(pipe)
+ {
+ }
+
+protected:
+ void run() override
+ {
+ RkISP1FrameInfo *info = data_->frameInfo_.find(frame());
+ if (!info)
+ LOG(RkISP1, Fatal) << "Frame not known";
+
+ if (info->paramFilled)
+ pipe_->param_->queueBuffer(info->paramBuffer);
+ else
+ LOG(RkISP1, Error)
+ << "Parameters not ready on time for frame "
+ << frame() << ", ignore parameters.";
+
+ pipe_->stat_->queueBuffer(info->statBuffer);
+ pipe_->video_->queueBuffer(info->videoBuffer);
+ }
+
+private:
+ RkISP1CameraData *data_;
+ PipelineHandlerRkISP1 *pipe_;
+};
+
+int RkISP1CameraData::loadIPA()
+{
+ ipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1);
+ if (!ipa_)
+ return -ENOENT;
+
+ ipa_->queueFrameAction.connect(this,
+ &RkISP1CameraData::queueFrameAction);
+
+ return 0;
+}
+
+void RkISP1CameraData::queueFrameAction(unsigned int frame,
+ const IPAOperationData &action)
+{
+ switch (action.operation) {
+ case RKISP1_IPA_ACTION_V4L2_SET: {
+ V4L2ControlList controls = action.v4l2controls[0];
+ timeline_.scheduleAction(utils::make_unique<RkISP1ActionSetSensor>(frame,
+ sensor_,
+ controls));
+ break;
+ }
+ case RKISP1_IPA_ACTION_PARAM_FILLED: {
+ RkISP1FrameInfo *info = frameInfo_.find(frame);
+ if (info)
+ info->paramFilled = true;
+ break;
+ }
+ case RKISP1_IPA_ACTION_METADATA:
+ metadataReady(frame, action.controls[0]);
+ break;
+ default:
+ LOG(RkISP1, Error) << "Unkown action " << action.operation;
+ break;
+ }
+}
+
+void RkISP1CameraData::metadataReady(unsigned int frame, const ControlList &metadata)
+{
+ PipelineHandlerRkISP1 *pipe =
+ static_cast<PipelineHandlerRkISP1 *>(pipe_);
+
+ RkISP1FrameInfo *info = frameInfo_.find(frame);
+ if (!info)
+ return;
+
+ info->request->metadata() = metadata;
+ info->metadataProcessed = true;
+
+ pipe->tryCompleteRequest(info->request);
+}
+
RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera,
RkISP1CameraData *data)
: CameraConfiguration()
@@ -202,12 +514,14 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()
PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)
: PipelineHandler(manager), dphy_(nullptr), isp_(nullptr),
- video_(nullptr)
+ video_(nullptr), param_(nullptr), stat_(nullptr)
{
}
PipelineHandlerRkISP1::~PipelineHandlerRkISP1()
{
+ delete param_;
+ delete stat_;
delete video_;
delete isp_;
delete dphy_;
@@ -324,6 +638,18 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
return -EINVAL;
}
+ V4L2DeviceFormat paramFormat = {};
+ paramFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS;
+ ret = param_->setFormat(&paramFormat);
+ if (ret)
+ return ret;
+
+ V4L2DeviceFormat statFormat = {};
+ statFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A;
+ ret = stat_->setFormat(&statFormat);
+ if (ret)
+ return ret;
+
cfg.setStream(&data->stream_);
return 0;
@@ -332,39 +658,135 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
int PipelineHandlerRkISP1::allocateBuffers(Camera *camera,
const std::set<Stream *> &streams)
{
+ RkISP1CameraData *data = cameraData(camera);
Stream *stream = *streams.begin();
+ int ret;
if (stream->memoryType() == InternalMemory)
- return video_->exportBuffers(&stream->bufferPool());
+ ret = video_->exportBuffers(&stream->bufferPool());
else
- return video_->importBuffers(&stream->bufferPool());
+ ret = video_->importBuffers(&stream->bufferPool());
+
+ if (ret)
+ return ret;
+
+ paramPool_.createBuffers(stream->configuration().bufferCount + 1);
+ ret = param_->exportBuffers(&paramPool_);
+ if (ret) {
+ video_->releaseBuffers();
+ return ret;
+ }
+
+ statPool_.createBuffers(stream->configuration().bufferCount + 1);
+ ret = stat_->exportBuffers(&statPool_);
+ if (ret) {
+ param_->releaseBuffers();
+ video_->releaseBuffers();
+ return ret;
+ }
+
+ for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {
+ data->ipaBuffers_.push_back({ .id = RKISP1_PARAM_BASE | i,
+ .memory = paramPool_.buffers()[i] });
+ paramBuffers_.push(new Buffer(i));
+ }
+
+ for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {
+ data->ipaBuffers_.push_back({ .id = RKISP1_STAT_BASE | i,
+ .memory = statPool_.buffers()[i] });
+ statBuffers_.push(new Buffer(i));
+ }
+
+ data->ipa_->mapBuffers(data->ipaBuffers_);
+
+ return ret;
}
int PipelineHandlerRkISP1::freeBuffers(Camera *camera,
const std::set<Stream *> &streams)
{
+ RkISP1CameraData *data = cameraData(camera);
+
+ while (!statBuffers_.empty()) {
+ delete statBuffers_.front();
+ statBuffers_.pop();
+ }
+
+ while (!paramBuffers_.empty()) {
+ delete paramBuffers_.front();
+ paramBuffers_.pop();
+ }
+
+ std::vector<unsigned int> ids;
+ for (IPABuffer &ipabuf : data->ipaBuffers_)
+ ids.push_back(ipabuf.id);
+
+ data->ipa_->unmapBuffers(ids);
+ data->ipaBuffers_.clear();
+
+ if (param_->releaseBuffers())
+ LOG(RkISP1, Error) << "Failed to release parameters buffers";
+
+ if (stat_->releaseBuffers())
+ LOG(RkISP1, Error) << "Failed to release stat buffers";
+
if (video_->releaseBuffers())
- LOG(RkISP1, Error) << "Failed to release buffers";
+ LOG(RkISP1, Error) << "Failed to release video buffers";
return 0;
}
int PipelineHandlerRkISP1::start(Camera *camera)
{
+ RkISP1CameraData *data = cameraData(camera);
int ret;
+ data->frame_ = 0;
+
+ ret = param_->streamOn();
+ if (ret) {
+ LOG(RkISP1, Error)
+ << "Failed to start parameters " << camera->name();
+ return ret;
+ }
+
+ ret = stat_->streamOn();
+ if (ret) {
+ param_->streamOff();
+ LOG(RkISP1, Error)
+ << "Failed to start statistics " << camera->name();
+ return ret;
+ }
+
ret = video_->streamOn();
- if (ret)
+ if (ret) {
+ param_->streamOff();
+ stat_->streamOff();
+
LOG(RkISP1, Error)
<< "Failed to start camera " << camera->name();
+ }
activeCamera_ = camera;
+ /* Inform IPA of stream configuration and sensor controls. */
+ std::map<unsigned int, IPAStream> streamConfig;
+ streamConfig[0] = {
+ .pixelFormat = data->stream_.configuration().pixelFormat,
+ .size = data->stream_.configuration().size,
+ };
+
+ std::map<unsigned int, V4L2ControlInfoMap> entityControls;
+ entityControls[0] = data->sensor_->controls();
+
+ data->ipa_->configure(streamConfig, entityControls);
+
return ret;
}
void PipelineHandlerRkISP1::stop(Camera *camera)
{
+ RkISP1CameraData *data = cameraData(camera);
int ret;
ret = video_->streamOff();
@@ -372,6 +794,18 @@ void PipelineHandlerRkISP1::stop(Camera *camera)
LOG(RkISP1, Warning)
<< "Failed to stop camera " << camera->name();
+ ret = stat_->streamOff();
+ if (ret)
+ LOG(RkISP1, Warning)
+ << "Failed to stop statistics " << camera->name();
+
+ ret = param_->streamOff();
+ if (ret)
+ LOG(RkISP1, Warning)
+ << "Failed to stop parameters " << camera->name();
+
+ data->timeline_.reset();
+
activeCamera_ = nullptr;
}
@@ -380,18 +814,24 @@ int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request)
RkISP1CameraData *data = cameraData(camera);
Stream *stream = &data->stream_;
- Buffer *buffer = request->findBuffer(stream);
- if (!buffer) {
- LOG(RkISP1, Error)
- << "Attempt to queue request with invalid stream";
+ PipelineHandler::queueRequest(camera, request);
+
+ RkISP1FrameInfo *info = data->frameInfo_.create(data->frame_, request,
+ stream);
+ if (!info)
return -ENOENT;
- }
- int ret = video_->queueBuffer(buffer);
- if (ret < 0)
- return ret;
+ IPAOperationData op;
+ op.operation = RKISP1_IPA_EVENT_QUEUE_REQUEST;
+ op.data = { data->frame_, RKISP1_PARAM_BASE | info->paramBuffer->index() };
+ op.controls = { request->controls() };
+ data->ipa_->processEvent(op);
- PipelineHandler::queueRequest(camera, request);
+ data->timeline_.scheduleAction(utils::make_unique<RkISP1ActionQueueBuffers>(data->frame_,
+ data,
+ this));
+
+ data->frame_++;
return 0;
}
@@ -435,11 +875,19 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)
std::unique_ptr<RkISP1CameraData> data =
utils::make_unique<RkISP1CameraData>(this);
+ data->controlInfo_.emplace(std::piecewise_construct,
+ std::forward_as_tuple(&controls::AeEnable),
+ std::forward_as_tuple(false, true));
+
data->sensor_ = new CameraSensor(sensor);
ret = data->sensor_->init();
if (ret)
return ret;
+ ret = data->loadIPA();
+ if (ret)
+ return ret;
+
std::set<Stream *> streams{ &data->stream_ };
std::shared_ptr<Camera> camera =
Camera::create(this, sensor->name(), streams);
@@ -478,7 +926,17 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
if (video_->open() < 0)
return false;
+ stat_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-statistics");
+ if (stat_->open() < 0)
+ return false;
+
+ param_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-input-params");
+ if (param_->open() < 0)
+ return false;
+
video_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady);
+ stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);
+ param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);
/* Configure default links. */
if (initLinks() < 0) {
@@ -504,13 +962,69 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
* Buffer Handling
*/
+void PipelineHandlerRkISP1::tryCompleteRequest(Request *request)
+{
+ RkISP1CameraData *data = cameraData(activeCamera_);
+ RkISP1FrameInfo *info = data->frameInfo_.find(request);
+ if (!info)
+ return;
+
+ if (request->hasPendingBuffers())
+ return;
+
+ if (!info->metadataProcessed)
+ return;
+
+ if (!info->paramDequeued)
+ return;
+
+ completeRequest(activeCamera_, request);
+
+ data->frameInfo_.destroy(info->frame);
+}
+
void PipelineHandlerRkISP1::bufferReady(Buffer *buffer)
{
ASSERT(activeCamera_);
+ RkISP1CameraData *data = cameraData(activeCamera_);
Request *request = buffer->request();
+ data->timeline_.bufferReady(buffer);
+
+ if (data->frame_ <= buffer->sequence())
+ data->frame_ = buffer->sequence() + 1;
+
completeBuffer(activeCamera_, request, buffer);
- completeRequest(activeCamera_, request);
+ tryCompleteRequest(request);
+}
+
+void PipelineHandlerRkISP1::paramReady(Buffer *buffer)
+{
+ ASSERT(activeCamera_);
+ RkISP1CameraData *data = cameraData(activeCamera_);
+
+ RkISP1FrameInfo *info = data->frameInfo_.find(buffer);
+
+ info->paramDequeued = true;
+ tryCompleteRequest(info->request);
+}
+
+void PipelineHandlerRkISP1::statReady(Buffer *buffer)
+{
+ ASSERT(activeCamera_);
+ RkISP1CameraData *data = cameraData(activeCamera_);
+
+ RkISP1FrameInfo *info = data->frameInfo_.find(buffer);
+ if (!info)
+ return;
+
+ unsigned int frame = info->frame;
+ unsigned int statid = RKISP1_STAT_BASE | info->statBuffer->index();
+
+ IPAOperationData op;
+ op.operation = RKISP1_IPA_EVENT_SIGNAL_STAT_BUFFER;
+ op.data = { frame, statid };
+ data->ipa_->processEvent(op);
}
REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1);
diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp
new file mode 100644
index 00000000..b98a1668
--- /dev/null
+++ b/src/libcamera/pipeline/rkisp1/timeline.cpp
@@ -0,0 +1,227 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2019, Google Inc.
+ *
+ * timeline.cpp - Timeline for per-frame control
+ */
+
+#include "timeline.h"
+
+#include "log.h"
+
+/**
+ * \file timeline.h
+ * \brief Timeline for per-frame control
+ */
+
+namespace libcamera {
+
+LOG_DEFINE_CATEGORY(Timeline)
+
+/**
+ * \class FrameAction
+ * \brief Action that can be schedule on a Timeline
+ *
+ * A frame action is an event schedule to be executed on a Timeline. A frame
+ * action has two primal attributes a frame number and a type.
+ *
+ * The frame number describes the frame to which the action is associated. The
+ * type is a numerical ID which identifies the action within the pipeline and
+ * IPA protocol.
+ */
+
+/**
+ * \class Timeline
+ * \brief Executor of FrameAction
+ *
+ * The timeline has three primary functions:
+ *
+ * 1. Keep track of the Start of Exposure (SOE) for every frame processed by
+ * the hardware. Using this information it shall keep an up-to-date estimate
+ * of the frame interval (time between two consecutive SOE events).
+ *
+ * The estimated frame interval together with recorded SOE events are the
+ * foundation for how the timeline schedule FrameAction at specific points
+ * in time.
+ * \todo Improve the frame interval estimation algorithm.
+ *
+ * 2. Keep track of current delays for different types of actions. The delays
+ * for different actions might differ during a capture session. Exposure time
+ * effects the over all FPS and different ISP parameters might impacts its
+ * processing time.
+ *
+ * The action type delays shall be updated by the IPA in conjunction with
+ * how it changes the capture parameters.
+ *
+ * 3. Schedule actions on the timeline. This is the process of taking a
+ * FrameAction which contains an abstract description of what frame and
+ * what type of action it contains and turning that into an time point
+ * and make sure the action is executed at that time.
+ */
+
+Timeline::Timeline()
+ : frameInterval_(0)
+{
+ timer_.timeout.connect(this, &Timeline::timeout);
+}
+
+/**
+ * \brief Reset and stop the timeline
+ *
+ * The timeline needs to be reset when the timeline should no longer execute
+ * actions. A timeline should be reset between two capture sessions to prevent
+ * the old capture session to effect the second one.
+ */
+void Timeline::reset()
+{
+ timer_.stop();
+
+ actions_.clear();
+ history_.clear();
+}
+
+/**
+ * \brief Schedule an action on the timeline
+ * \param[in] action FrameAction to schedule
+ *
+ * The act of scheduling an action to the timeline is the process of taking
+ * the properties of the action (type, frame and time offsets) and translating
+ * that to a time point using the current values for the action type timings
+ * value recorded in the timeline. If an action is scheduled too late, execute
+ * it immediately.
+ */
+void Timeline::scheduleAction(std::unique_ptr<FrameAction> action)
+{
+ unsigned int lastFrame;
+ utils::time_point lastTime;
+
+ if (history_.empty()) {
+ lastFrame = 0;
+ lastTime = std::chrono::steady_clock::now();
+ } else {
+ lastFrame = history_.back().first;
+ lastTime = history_.back().second;
+ }
+
+ /*
+ * Calculate when the action shall be schedule by first finding out how
+ * many frames in the future the action acts on and then add the actions
+ * frame offset. After the spatial frame offset is found out translate
+ * that to a time point by using the last estimated start of exposure
+ * (SOE) as the fixed offset. Lastly add the action time offset to the
+ * time point.
+ */
+ int frame = action->frame() - lastFrame + frameOffset(action->type());
+ utils::time_point deadline = lastTime + frame * frameInterval_
+ + timeOffset(action->type());
+
+ utils::time_point now = std::chrono::steady_clock::now();
+ if (deadline < now) {
+ LOG(Timeline, Warning)
+ << "Action scheduled too late "
+ << utils::time_point_to_string(deadline)
+ << ", run now " << utils::time_point_to_string(now);
+ action->run();
+ } else {
+ actions_.insert({ deadline, std::move(action) });
+ updateDeadline();
+ }
+}
+
+void Timeline::notifyStartOfExposure(unsigned int frame, utils::time_point time)
+{
+ history_.push_back(std::make_pair(frame, time));
+
+ if (history_.size() <= HISTORY_DEPTH / 2)
+ return;
+
+ while (history_.size() > HISTORY_DEPTH)
+ history_.pop_front();
+
+ /* Update esitmated time between two start of exposures. */
+ utils::duration sumExposures(0);
+ unsigned int numExposures = 0;
+
+ utils::time_point lastTime;
+ for (auto it = history_.begin(); it != history_.end(); it++) {
+ if (it != history_.begin()) {
+ sumExposures += it->second - lastTime;
+ numExposures++;
+ }
+
+ lastTime = it->second;
+ }
+
+ frameInterval_ = sumExposures;
+ if (numExposures)
+ frameInterval_ /= numExposures;
+}
+
+int Timeline::frameOffset(unsigned int type) const
+{
+ const auto it = delays_.find(type);
+ if (it == delays_.end()) {
+ LOG(Timeline, Error)
+ << "No frame offset set for action type " << type;
+ return 0;
+ }
+
+ return it->second.first;
+}
+
+utils::duration Timeline::timeOffset(unsigned int type) const
+{
+ const auto it = delays_.find(type);
+ if (it == delays_.end()) {
+ LOG(Timeline, Error)
+ << "No time offset set for action type " << type;
+ return utils::duration::zero();
+ }
+
+ return it->second.second;
+}
+
+void Timeline::setRawDelay(unsigned int type, int frame, utils::duration time)
+{
+ delays_[type] = std::make_pair(frame, time);
+}
+
+void Timeline::updateDeadline()
+{
+ if (actions_.empty())
+ return;
+
+ const utils::time_point &deadline = actions_.begin()->first;
+
+ if (timer_.isRunning() && deadline >= timer_.deadline())
+ return;
+
+ if (deadline <= std::chrono::steady_clock::now()) {
+ timeout(&timer_);
+ return;
+ }
+
+ timer_.start(deadline);
+}
+
+void Timeline::timeout(Timer *timer)
+{
+ utils::time_point now = std::chrono::steady_clock::now();
+
+ for (auto it = actions_.begin(); it != actions_.end();) {
+ const utils::time_point &sched = it->first;
+
+ if (sched > now)
+ break;
+
+ FrameAction *action = it->second.get();
+
+ action->run();
+
+ it = actions_.erase(it);
+ }
+
+ updateDeadline();
+}
+
+} /* namespace libcamera */
diff --git a/src/libcamera/pipeline/rkisp1/timeline.h b/src/libcamera/pipeline/rkisp1/timeline.h
new file mode 100644
index 00000000..9d30e4ea
--- /dev/null
+++ b/src/libcamera/pipeline/rkisp1/timeline.h
@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2019, Google Inc.
+ *
+ * timeline.h - Timeline for per-frame controls
+ */
+#ifndef __LIBCAMERA_TIMELINE_H__
+#define __LIBCAMERA_TIMELINE_H__
+
+#include <list>
+#include <map>
+
+#include <libcamera/timer.h>
+
+#include "utils.h"
+
+namespace libcamera {
+
+class FrameAction
+{
+public:
+ FrameAction(unsigned int frame, unsigned int type)
+ : frame_(frame), type_(type) {}
+
+ virtual ~FrameAction() {}
+
+ unsigned int frame() const { return frame_; }
+ unsigned int type() const { return type_; }
+
+ virtual void run() = 0;
+
+private:
+ unsigned int frame_;
+ unsigned int type_;
+};
+
+class Timeline
+{
+public:
+ Timeline();
+ virtual ~Timeline() {}
+
+ virtual void reset();
+ virtual void scheduleAction(std::unique_ptr<FrameAction> action);
+ virtual void notifyStartOfExposure(unsigned int frame, utils::time_point time);
+
+ utils::duration frameInterval() const { return frameInterval_; }
+
+protected:
+ int frameOffset(unsigned int type) const;
+ utils::duration timeOffset(unsigned int type) const;
+
+ void setRawDelay(unsigned int type, int frame, utils::duration time);
+
+ std::map<unsigned int, std::pair<int, utils::duration>> delays_;
+
+private:
+ static constexpr unsigned int HISTORY_DEPTH = 10;
+
+ void timeout(Timer *timer);
+ void updateDeadline();
+
+ std::list<std::pair<unsigned int, utils::time_point>> history_;
+ std::multimap<utils::time_point, std::unique_ptr<FrameAction>> actions_;
+ utils::duration frameInterval_;
+
+ Timer timer_;
+};
+
+} /* namespace libcamera */
+
+#endif /* __LIBCAMERA_TIMELINE_H__ */