4 files changed, 831 insertions, 17 deletions

diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build
index f1cc4046..d04fb452 100644
--- a/src/libcamera/pipeline/rkisp1/meson.build
+++ b/src/libcamera/pipeline/rkisp1/meson.build
@@ -1,3 +1,4 @@
 libcamera_sources += files([
     'rkisp1.cpp',
+    'timeline.cpp',
 ])
diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
index de4ab523..029d5868 100644
--- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp
+++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
@@ -9,32 +9,115 @@
 #include <array>
 #include <iomanip>
 #include <memory>
-#include <vector>
+#include <queue>
 
 #include <linux/media-bus-format.h>
 
+#include <ipa/rkisp1.h>
+#include <libcamera/buffer.h>
 #include <libcamera/camera.h>
+#include <libcamera/control_ids.h>
 #include <libcamera/request.h>
 #include <libcamera/stream.h>
 
 #include "camera_sensor.h"
 #include "device_enumerator.h"
+#include "ipa_manager.h"
 #include "log.h"
 #include "media_device.h"
 #include "pipeline_handler.h"
+#include "timeline.h"
 #include "utils.h"
 #include "v4l2_subdevice.h"
 #include "v4l2_videodevice.h"
 
+#define RKISP1_PARAM_BASE 0x100
+#define RKISP1_STAT_BASE 0x200
+
 namespace libcamera {
 
 LOG_DEFINE_CATEGORY(RkISP1)
 
+class PipelineHandlerRkISP1;
+class RkISP1ActionQueueBuffers;
+
+enum RkISP1ActionType {
+	SetSensor,
+	SOE,
+	QueueBuffers,
+};
+
+struct RkISP1FrameInfo {
+	unsigned int frame;
+	Request *request;
+
+	Buffer *paramBuffer;
+	Buffer *statBuffer;
+	Buffer *videoBuffer;
+
+	bool paramFilled;
+	bool paramDequeued;
+	bool metadataProcessed;
+};
+
+class RkISP1Frames
+{
+public:
+	RkISP1Frames(PipelineHandler *pipe);
+
+	RkISP1FrameInfo *create(unsigned int frame, Request *request, Stream *stream);
+	int destroy(unsigned int frame);
+
+	RkISP1FrameInfo *find(unsigned int frame);
+	RkISP1FrameInfo *find(Buffer *buffer);
+	RkISP1FrameInfo *find(Request *request);
+
+private:
+	PipelineHandlerRkISP1 *pipe_;
+	std::map<unsigned int, RkISP1FrameInfo *> frameInfo_;
+};
+
+class RkISP1Timeline : public Timeline
+{
+public:
+	RkISP1Timeline()
+		: Timeline()
+	{
+		setDelay(SetSensor, -1, 5);
+		setDelay(SOE, 0, -1);
+		setDelay(QueueBuffers, -1, 10);
+	}
+
+	void bufferReady(Buffer *buffer)
+	{
+		/*
+		 * Calculate SOE by taking the end of DMA set by the kernel and applying
+		 * the time offsets provideprovided by the IPA to find the best estimate
+		 * of SOE.
+		 */
+
+		ASSERT(frameOffset(SOE) == 0);
+
+		utils::time_point soe = std::chrono::time_point<utils::clock>()
+			+ std::chrono::nanoseconds(buffer->timestamp())
+			+ timeOffset(SOE);
+
+		notifyStartOfExposure(buffer->sequence(), soe);
+	}
+
+	void setDelay(unsigned int type, int frame, int msdelay)
+	{
+		utils::duration delay = std::chrono::milliseconds(msdelay);
+		setRawDelay(type, frame, delay);
+	}
+};
+
 class RkISP1CameraData : public CameraData
 {
 public:
 	RkISP1CameraData(PipelineHandler *pipe)
-		: CameraData(pipe), sensor_(nullptr)
+		: CameraData(pipe), sensor_(nullptr), frame_(0),
+		  frameInfo_(pipe)
 	{
 	}
 
@@ -43,8 +126,20 @@ public:
 		delete sensor_;
 	}
 
+	int loadIPA();
+
 	Stream stream_;
 	CameraSensor *sensor_;
+	unsigned int frame_;
+	std::vector<IPABuffer> ipaBuffers_;
+	RkISP1Frames frameInfo_;
+	RkISP1Timeline timeline_;
+
+private:
+	void queueFrameAction(unsigned int frame,
+			      const IPAOperationData &action);
+
+	void metadataReady(unsigned int frame, const ControlList &metadata);
 };
 
 class RkISP1CameraConfiguration : public CameraConfiguration
@@ -99,18 +194,235 @@ private:
 			PipelineHandler::cameraData(camera));
 	}
 
+	friend RkISP1ActionQueueBuffers;
+	friend RkISP1CameraData;
+	friend RkISP1Frames;
+
 	int initLinks();
 	int createCamera(MediaEntity *sensor);
+	void tryCompleteRequest(Request *request);
 	void bufferReady(Buffer *buffer);
+	void paramReady(Buffer *buffer);
+	void statReady(Buffer *buffer);
 
 	MediaDevice *media_;
 	V4L2Subdevice *dphy_;
 	V4L2Subdevice *isp_;
 	V4L2VideoDevice *video_;
+	V4L2VideoDevice *param_;
+	V4L2VideoDevice *stat_;
+
+	BufferPool paramPool_;
+	BufferPool statPool_;
+
+	std::queue<Buffer *> paramBuffers_;
+	std::queue<Buffer *> statBuffers_;
 
 	Camera *activeCamera_;
 };
 
+RkISP1Frames::RkISP1Frames(PipelineHandler *pipe)
+	: pipe_(dynamic_cast<PipelineHandlerRkISP1 *>(pipe))
+{
+}
+
+RkISP1FrameInfo *RkISP1Frames::create(unsigned int frame, Request *request, Stream *stream)
+{
+	if (pipe_->paramBuffers_.empty()) {
+		LOG(RkISP1, Error) << "Parameters buffer underrun";
+		return nullptr;
+	}
+	Buffer *paramBuffer = pipe_->paramBuffers_.front();
+
+	if (pipe_->statBuffers_.empty()) {
+		LOG(RkISP1, Error) << "Statisitc buffer underrun";
+		return nullptr;
+	}
+	Buffer *statBuffer = pipe_->statBuffers_.front();
+
+	Buffer *videoBuffer = request->findBuffer(stream);
+	if (!videoBuffer) {
+		LOG(RkISP1, Error)
+			<< "Attempt to queue request with invalid stream";
+		return nullptr;
+	}
+
+	pipe_->paramBuffers_.pop();
+	pipe_->statBuffers_.pop();
+
+	RkISP1FrameInfo *info = new RkISP1FrameInfo;
+
+	info->frame = frame;
+	info->request = request;
+	info->paramBuffer = paramBuffer;
+	info->videoBuffer = videoBuffer;
+	info->statBuffer = statBuffer;
+	info->paramFilled = false;
+	info->paramDequeued = false;
+	info->metadataProcessed = false;
+
+	frameInfo_[frame] = info;
+
+	return info;
+}
+
+int RkISP1Frames::destroy(unsigned int frame)
+{
+	RkISP1FrameInfo *info = find(frame);
+	if (!info)
+		return -ENOENT;
+
+	pipe_->paramBuffers_.push(info->paramBuffer);
+	pipe_->statBuffers_.push(info->statBuffer);
+
+	frameInfo_.erase(info->frame);
+
+	delete info;
+
+	return 0;
+}
+
+RkISP1FrameInfo *RkISP1Frames::find(unsigned int frame)
+{
+	auto itInfo = frameInfo_.find(frame);
+
+	if (itInfo != frameInfo_.end())
+		return itInfo->second;
+
+	LOG(RkISP1, Error) << "Can't locate info from frame";
+	return nullptr;
+}
+
+RkISP1FrameInfo *RkISP1Frames::find(Buffer *buffer)
+{
+	for (auto &itInfo : frameInfo_) {
+		RkISP1FrameInfo *info = itInfo.second;
+
+		if (info->paramBuffer == buffer ||
+		    info->statBuffer == buffer ||
+		    info->videoBuffer == buffer)
+			return info;
+	}
+
+	LOG(RkISP1, Error) << "Can't locate info from buffer";
+	return nullptr;
+}
+
+RkISP1FrameInfo *RkISP1Frames::find(Request *request)
+{
+	for (auto &itInfo : frameInfo_) {
+		RkISP1FrameInfo *info = itInfo.second;
+
+		if (info->request == request)
+			return info;
+	}
+
+	LOG(RkISP1, Error) << "Can't locate info from request";
+	return nullptr;
+}
+
+class RkISP1ActionSetSensor : public FrameAction
+{
+public:
+	RkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls)
+		: FrameAction(frame, SetSensor), sensor_(sensor), controls_(controls) {}
+
+protected:
+	void run() override
+	{
+		sensor_->setControls(&controls_);
+	}
+
+private:
+	CameraSensor *sensor_;
+	V4L2ControlList controls_;
+};
+
+class RkISP1ActionQueueBuffers : public FrameAction
+{
+public:
+	RkISP1ActionQueueBuffers(unsigned int frame, RkISP1CameraData *data,
+				 PipelineHandlerRkISP1 *pipe)
+		: FrameAction(frame, QueueBuffers), data_(data), pipe_(pipe)
+	{
+	}
+
+protected:
+	void run() override
+	{
+		RkISP1FrameInfo *info = data_->frameInfo_.find(frame());
+		if (!info)
+			LOG(RkISP1, Fatal) << "Frame not known";
+
+		if (info->paramFilled)
+			pipe_->param_->queueBuffer(info->paramBuffer);
+		else
+			LOG(RkISP1, Error)
+				<< "Parameters not ready on time for frame "
+				<< frame() << ", ignore parameters.";
+
+		pipe_->stat_->queueBuffer(info->statBuffer);
+		pipe_->video_->queueBuffer(info->videoBuffer);
+	}
+
+private:
+	RkISP1CameraData *data_;
+	PipelineHandlerRkISP1 *pipe_;
+};
+
+int RkISP1CameraData::loadIPA()
+{
+	ipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1);
+	if (!ipa_)
+		return -ENOENT;
+
+	ipa_->queueFrameAction.connect(this,
+				       &RkISP1CameraData::queueFrameAction);
+
+	return 0;
+}
+
+void RkISP1CameraData::queueFrameAction(unsigned int frame,
+					const IPAOperationData &action)
+{
+	switch (action.operation) {
+	case RKISP1_IPA_ACTION_V4L2_SET: {
+		V4L2ControlList controls = action.v4l2controls[0];
+		timeline_.scheduleAction(utils::make_unique<RkISP1ActionSetSensor>(frame,
+										   sensor_,
+										   controls));
+		break;
+	}
+	case RKISP1_IPA_ACTION_PARAM_FILLED: {
+		RkISP1FrameInfo *info = frameInfo_.find(frame);
+		if (info)
+			info->paramFilled = true;
+		break;
+	}
+	case RKISP1_IPA_ACTION_METADATA:
+		metadataReady(frame, action.controls[0]);
+		break;
+	default:
+		LOG(RkISP1, Error) << "Unkown action " << action.operation;
+		break;
+	}
+}
+
+void RkISP1CameraData::metadataReady(unsigned int frame, const ControlList &metadata)
+{
+	PipelineHandlerRkISP1 *pipe =
+		static_cast<PipelineHandlerRkISP1 *>(pipe_);
+
+	RkISP1FrameInfo *info = frameInfo_.find(frame);
+	if (!info)
+		return;
+
+	info->request->metadata() = metadata;
+	info->metadataProcessed = true;
+
+	pipe->tryCompleteRequest(info->request);
+}
+
 RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera,
 						     RkISP1CameraData *data)
 	: CameraConfiguration()
@@ -202,12 +514,14 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()
 
 PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)
 	: PipelineHandler(manager), dphy_(nullptr), isp_(nullptr),
-	  video_(nullptr)
+	  video_(nullptr), param_(nullptr), stat_(nullptr)
 {
 }
 
 PipelineHandlerRkISP1::~PipelineHandlerRkISP1()
 {
+	delete param_;
+	delete stat_;
 	delete video_;
 	delete isp_;
 	delete dphy_;
@@ -324,6 +638,18 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
 		return -EINVAL;
 	}
 
+	V4L2DeviceFormat paramFormat = {};
+	paramFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS;
+	ret = param_->setFormat(&paramFormat);
+	if (ret)
+		return ret;
+
+	V4L2DeviceFormat statFormat = {};
+	statFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A;
+	ret = stat_->setFormat(&statFormat);
+	if (ret)
+		return ret;
+
 	cfg.setStream(&data->stream_);
 
 	return 0;
@@ -332,39 +658,135 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
 int PipelineHandlerRkISP1::allocateBuffers(Camera *camera,
 					   const std::set<Stream *> &streams)
 {
+	RkISP1CameraData *data = cameraData(camera);
 	Stream *stream = *streams.begin();
+	int ret;
 
 	if (stream->memoryType() == InternalMemory)
-		return video_->exportBuffers(&stream->bufferPool());
+		ret = video_->exportBuffers(&stream->bufferPool());
 	else
-		return video_->importBuffers(&stream->bufferPool());
+		ret = video_->importBuffers(&stream->bufferPool());
+
+	if (ret)
+		return ret;
+
+	paramPool_.createBuffers(stream->configuration().bufferCount + 1);
+	ret = param_->exportBuffers(&paramPool_);
+	if (ret) {
+		video_->releaseBuffers();
+		return ret;
+	}
+
+	statPool_.createBuffers(stream->configuration().bufferCount + 1);
+	ret = stat_->exportBuffers(&statPool_);
+	if (ret) {
+		param_->releaseBuffers();
+		video_->releaseBuffers();
+		return ret;
+	}
+
+	for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {
+		data->ipaBuffers_.push_back({ .id = RKISP1_PARAM_BASE | i,
+					      .memory = paramPool_.buffers()[i] });
+		paramBuffers_.push(new Buffer(i));
+	}
+
+	for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {
+		data->ipaBuffers_.push_back({ .id = RKISP1_STAT_BASE | i,
+					      .memory = statPool_.buffers()[i] });
+		statBuffers_.push(new Buffer(i));
+	}
+
+	data->ipa_->mapBuffers(data->ipaBuffers_);
+
+	return ret;
 }
 
 int PipelineHandlerRkISP1::freeBuffers(Camera *camera,
 				       const std::set<Stream *> &streams)
 {
+	RkISP1CameraData *data = cameraData(camera);
+
+	while (!statBuffers_.empty()) {
+		delete statBuffers_.front();
+		statBuffers_.pop();
+	}
+
+	while (!paramBuffers_.empty()) {
+		delete paramBuffers_.front();
+		paramBuffers_.pop();
+	}
+
+	std::vector<unsigned int> ids;
+	for (IPABuffer &ipabuf : data->ipaBuffers_)
+		ids.push_back(ipabuf.id);
+
+	data->ipa_->unmapBuffers(ids);
+	data->ipaBuffers_.clear();
+
+	if (param_->releaseBuffers())
+		LOG(RkISP1, Error) << "Failed to release parameters buffers";
+
+	if (stat_->releaseBuffers())
+		LOG(RkISP1, Error) << "Failed to release stat buffers";
+
 	if (video_->releaseBuffers())
-		LOG(RkISP1, Error) << "Failed to release buffers";
+		LOG(RkISP1, Error) << "Failed to release video buffers";
 
 	return 0;
 }
 
 int PipelineHandlerRkISP1::start(Camera *camera)
 {
+	RkISP1CameraData *data = cameraData(camera);
 	int ret;
 
+	data->frame_ = 0;
+
+	ret = param_->streamOn();
+	if (ret) {
+		LOG(RkISP1, Error)
+			<< "Failed to start parameters " << camera->name();
+		return ret;
+	}
+
+	ret = stat_->streamOn();
+	if (ret) {
+		param_->streamOff();
+		LOG(RkISP1, Error)
+			<< "Failed to start statistics " << camera->name();
+		return ret;
+	}
+
 	ret = video_->streamOn();
-	if (ret)
+	if (ret) {
+		param_->streamOff();
+		stat_->streamOff();
+
 		LOG(RkISP1, Error)
 			<< "Failed to start camera " << camera->name();
+	}
 
 	activeCamera_ = camera;
 
+	/* Inform IPA of stream configuration and sensor controls. */
+	std::map<unsigned int, IPAStream> streamConfig;
+	streamConfig[0] = {
+		.pixelFormat = data->stream_.configuration().pixelFormat,
+		.size = data->stream_.configuration().size,
+	};
+
+	std::map<unsigned int, V4L2ControlInfoMap> entityControls;
+	entityControls[0] = data->sensor_->controls();
+
+	data->ipa_->configure(streamConfig, entityControls);
+
 	return ret;
 }
 
 void PipelineHandlerRkISP1::stop(Camera *camera)
 {
+	RkISP1CameraData *data = cameraData(camera);
 	int ret;
 
 	ret = video_->streamOff();
@@ -372,6 +794,18 @@ void PipelineHandlerRkISP1::stop(Camera *camera)
 		LOG(RkISP1, Warning)
 			<< "Failed to stop camera " << camera->name();
 
+	ret = stat_->streamOff();
+	if (ret)
+		LOG(RkISP1, Warning)
+			<< "Failed to stop statistics " << camera->name();
+
+	ret = param_->streamOff();
+	if (ret)
+		LOG(RkISP1, Warning)
+			<< "Failed to stop parameters " << camera->name();
+
+	data->timeline_.reset();
+
 	activeCamera_ = nullptr;
 }
 
@@ -380,18 +814,24 @@ int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request)
 	RkISP1CameraData *data = cameraData(camera);
 	Stream *stream = &data->stream_;
 
-	Buffer *buffer = request->findBuffer(stream);
-	if (!buffer) {
-		LOG(RkISP1, Error)
-			<< "Attempt to queue request with invalid stream";
+	PipelineHandler::queueRequest(camera, request);
+
+	RkISP1FrameInfo *info = data->frameInfo_.create(data->frame_, request,
+							stream);
+	if (!info)
 		return -ENOENT;
-	}
 
-	int ret = video_->queueBuffer(buffer);
-	if (ret < 0)
-		return ret;
+	IPAOperationData op;
+	op.operation = RKISP1_IPA_EVENT_QUEUE_REQUEST;
+	op.data = { data->frame_, RKISP1_PARAM_BASE | info->paramBuffer->index() };
+	op.controls = { request->controls() };
+	data->ipa_->processEvent(op);
 
-	PipelineHandler::queueRequest(camera, request);
+	data->timeline_.scheduleAction(utils::make_unique<RkISP1ActionQueueBuffers>(data->frame_,
+										    data,
+										    this));
+
+	data->frame_++;
 
 	return 0;
 }
@@ -435,11 +875,19 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)
 	std::unique_ptr<RkISP1CameraData> data =
 		utils::make_unique<RkISP1CameraData>(this);
 
+	data->controlInfo_.emplace(std::piecewise_construct,
+				   std::forward_as_tuple(&controls::AeEnable),
+				   std::forward_as_tuple(false, true));
+
 	data->sensor_ = new CameraSensor(sensor);
 	ret = data->sensor_->init();
 	if (ret)
 		return ret;
 
+	ret = data->loadIPA();
+	if (ret)
+		return ret;
+
 	std::set<Stream *> streams{ &data->stream_ };
 	std::shared_ptr<Camera> camera =
 		Camera::create(this, sensor->name(), streams);
@@ -478,7 +926,17 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
 	if (video_->open() < 0)
 		return false;
 
+	stat_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-statistics");
+	if (stat_->open() < 0)
+		return false;
+
+	param_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-input-params");
+	if (param_->open() < 0)
+		return false;
+
 	video_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady);
+	stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);
+	param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);
 
 	/* Configure default links. */
 	if (initLinks() < 0) {
@@ -504,13 +962,69 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
  * Buffer Handling
  */
 
+void PipelineHandlerRkISP1::tryCompleteRequest(Request *request)
+{
+	RkISP1CameraData *data = cameraData(activeCamera_);
+	RkISP1FrameInfo *info = data->frameInfo_.find(request);
+	if (!info)
+		return;
+
+	if (request->hasPendingBuffers())
+		return;
+
+	if (!info->metadataProcessed)
+		return;
+
+	if (!info->paramDequeued)
+		return;
+
+	completeRequest(activeCamera_, request);
+
+	data->frameInfo_.destroy(info->frame);
+}
+
 void PipelineHandlerRkISP1::bufferReady(Buffer *buffer)
 {
 	ASSERT(activeCamera_);
+	RkISP1CameraData *data = cameraData(activeCamera_);
 	Request *request = buffer->request();
 
+	data->timeline_.bufferReady(buffer);
+
+	if (data->frame_ <= buffer->sequence())
+		data->frame_ = buffer->sequence() + 1;
+
 	completeBuffer(activeCamera_, request, buffer);
-	completeRequest(activeCamera_, request);
+	tryCompleteRequest(request);
+}
+
+void PipelineHandlerRkISP1::paramReady(Buffer *buffer)
+{
+	ASSERT(activeCamera_);
+	RkISP1CameraData *data = cameraData(activeCamera_);
+
+	RkISP1FrameInfo *info = data->frameInfo_.find(buffer);
+
+	info->paramDequeued = true;
+	tryCompleteRequest(info->request);
+}
+
+void PipelineHandlerRkISP1::statReady(Buffer *buffer)
+{
+	ASSERT(activeCamera_);
+	RkISP1CameraData *data = cameraData(activeCamera_);
+
+	RkISP1FrameInfo *info = data->frameInfo_.find(buffer);
+	if (!info)
+		return;
+
+	unsigned int frame = info->frame;
+	unsigned int statid = RKISP1_STAT_BASE | info->statBuffer->index();
+
+	IPAOperationData op;
+	op.operation = RKISP1_IPA_EVENT_SIGNAL_STAT_BUFFER;
+	op.data = { frame, statid };
+	data->ipa_->processEvent(op);
 }
 
 REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1);
diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp
new file mode 100644
index 00000000..b98a1668
--- /dev/null
+++ b/src/libcamera/pipeline/rkisp1/timeline.cpp
@@ -0,0 +1,227 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2019, Google Inc.
+ *
+ * timeline.cpp - Timeline for per-frame control
+ */
+
+#include "timeline.h"
+
+#include "log.h"
+
+/**
+ * \file timeline.h
+ * \brief Timeline for per-frame control
+ */
+
+namespace libcamera {
+
+LOG_DEFINE_CATEGORY(Timeline)
+
+/**
+ * \class FrameAction
+ * \brief Action that can be schedule on a Timeline
+ *
+ * A frame action is an event schedule to be executed on a Timeline. A frame
+ * action has two primal attributes a frame number and a type.
+ *
+ * The frame number describes the frame to which the action is associated. The
+ * type is a numerical ID which identifies the action within the pipeline and
+ * IPA protocol.
+ */
+
+/**
+ * \class Timeline
+ * \brief Executor of FrameAction
+ *
+ * The timeline has three primary functions:
+ *
+ * 1. Keep track of the Start of Exposure (SOE) for every frame processed by
+ *    the hardware. Using this information it shall keep an up-to-date estimate
+ *    of the frame interval (time between two consecutive SOE events).
+ *
+ *    The estimated frame interval together with recorded SOE events are the
+ *    foundation for how the timeline schedule FrameAction at specific points
+ *    in time.
+ *    \todo Improve the frame interval estimation algorithm.
+ *
+ * 2. Keep track of current delays for different types of actions. The delays
+ *    for different actions might differ during a capture session. Exposure time
+ *    effects the over all FPS and different ISP parameters might impacts its
+ *    processing time.
+ *
+ *    The action type delays shall be updated by the IPA in conjunction with
+ *    how it changes the capture parameters.
+ *
+ * 3. Schedule actions on the timeline. This is the process of taking a
+ *    FrameAction which contains an abstract description of what frame and
+ *    what type of action it contains and turning that into an time point
+ *    and make sure the action is executed at that time.
+ */
+
+Timeline::Timeline()
+	: frameInterval_(0)
+{
+	timer_.timeout.connect(this, &Timeline::timeout);
+}
+
+/**
+ * \brief Reset and stop the timeline
+ *
+ * The timeline needs to be reset when the timeline should no longer execute
+ * actions. A timeline should be reset between two capture sessions to prevent
+ * the old capture session to effect the second one.
+ */
+void Timeline::reset()
+{
+	timer_.stop();
+
+	actions_.clear();
+	history_.clear();
+}
+
+/**
+ * \brief Schedule an action on the timeline
+ * \param[in] action FrameAction to schedule
+ *
+ * The act of scheduling an action to the timeline is the process of taking
+ * the properties of the action (type, frame and time offsets) and translating
+ * that to a time point using the current values for the action type timings
+ * value recorded in the timeline. If an action is scheduled too late, execute
+ * it immediately.
+ */
+void Timeline::scheduleAction(std::unique_ptr<FrameAction> action)
+{
+	unsigned int lastFrame;
+	utils::time_point lastTime;
+
+	if (history_.empty()) {
+		lastFrame = 0;
+		lastTime = std::chrono::steady_clock::now();
+	} else {
+		lastFrame = history_.back().first;
+		lastTime = history_.back().second;
+	}
+
+	/*
+	 * Calculate when the action shall be schedule by first finding out how
+	 * many frames in the future the action acts on and then add the actions
+	 * frame offset. After the spatial frame offset is found out translate
+	 * that to a time point by using the last estimated start of exposure
+	 * (SOE) as the fixed offset. Lastly add the action time offset to the
+	 * time point.
+	 */
+	int frame = action->frame() - lastFrame + frameOffset(action->type());
+	utils::time_point deadline = lastTime + frame * frameInterval_
+		+ timeOffset(action->type());
+
+	utils::time_point now = std::chrono::steady_clock::now();
+	if (deadline < now) {
+		LOG(Timeline, Warning)
+			<< "Action scheduled too late "
+			<< utils::time_point_to_string(deadline)
+			<< ", run now " << utils::time_point_to_string(now);
+		action->run();
+	} else {
+		actions_.insert({ deadline, std::move(action) });
+		updateDeadline();
+	}
+}
+
+void Timeline::notifyStartOfExposure(unsigned int frame, utils::time_point time)
+{
+	history_.push_back(std::make_pair(frame, time));
+
+	if (history_.size() <= HISTORY_DEPTH / 2)
+		return;
+
+	while (history_.size() > HISTORY_DEPTH)
+		history_.pop_front();
+
+	/* Update esitmated time between two start of exposures. */
+	utils::duration sumExposures(0);
+	unsigned int numExposures = 0;
+
+	utils::time_point lastTime;
+	for (auto it = history_.begin(); it != history_.end(); it++) {
+		if (it != history_.begin()) {
+			sumExposures += it->second - lastTime;
+			numExposures++;
+		}
+
+		lastTime = it->second;
+	}
+
+	frameInterval_ = sumExposures;
+	if (numExposures)
+		frameInterval_ /= numExposures;
+}
+
+int Timeline::frameOffset(unsigned int type) const
+{
+	const auto it = delays_.find(type);
+	if (it == delays_.end()) {
+		LOG(Timeline, Error)
+			<< "No frame offset set for action type " << type;
+		return 0;
+	}
+
+	return it->second.first;
+}
+
+utils::duration Timeline::timeOffset(unsigned int type) const
+{
+	const auto it = delays_.find(type);
+	if (it == delays_.end()) {
+		LOG(Timeline, Error)
+			<< "No time offset set for action type " << type;
+		return utils::duration::zero();
+	}
+
+	return it->second.second;
+}
+
+void Timeline::setRawDelay(unsigned int type, int frame, utils::duration time)
+{
+	delays_[type] = std::make_pair(frame, time);
+}
+
+void Timeline::updateDeadline()
+{
+	if (actions_.empty())
+		return;
+
+	const utils::time_point &deadline = actions_.begin()->first;
+
+	if (timer_.isRunning() && deadline >= timer_.deadline())
+		return;
+
+	if (deadline <= std::chrono::steady_clock::now()) {
+		timeout(&timer_);
+		return;
+	}
+
+	timer_.start(deadline);
+}
+
+void Timeline::timeout(Timer *timer)
+{
+	utils::time_point now = std::chrono::steady_clock::now();
+
+	for (auto it = actions_.begin(); it != actions_.end();) {
+		const utils::time_point &sched = it->first;
+
+		if (sched > now)
+			break;
+
+		FrameAction *action = it->second.get();
+
+		action->run();
+
+		it = actions_.erase(it);
+	}
+
+	updateDeadline();
+}
+
+} /* namespace libcamera */
diff --git a/src/libcamera/pipeline/rkisp1/timeline.h b/src/libcamera/pipeline/rkisp1/timeline.h
new file mode 100644
index 00000000..9d30e4ea
--- /dev/null
+++ b/src/libcamera/pipeline/rkisp1/timeline.h
@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2019, Google Inc.
+ *
+ * timeline.h - Timeline for per-frame controls
+ */
+#ifndef __LIBCAMERA_TIMELINE_H__
+#define __LIBCAMERA_TIMELINE_H__
+
+#include <list>
+#include <map>
+
+#include <libcamera/timer.h>
+
+#include "utils.h"
+
+namespace libcamera {
+
+class FrameAction
+{
+public:
+	FrameAction(unsigned int frame, unsigned int type)
+		: frame_(frame), type_(type) {}
+
+	virtual ~FrameAction() {}
+
+	unsigned int frame() const { return frame_; }
+	unsigned int type() const { return type_; }
+
+	virtual void run() = 0;
+
+private:
+	unsigned int frame_;
+	unsigned int type_;
+};
+
+class Timeline
+{
+public:
+	Timeline();
+	virtual ~Timeline() {}
+
+	virtual void reset();
+	virtual void scheduleAction(std::unique_ptr<FrameAction> action);
+	virtual void notifyStartOfExposure(unsigned int frame, utils::time_point time);
+
+	utils::duration frameInterval() const { return frameInterval_; }
+
+protected:
+	int frameOffset(unsigned int type) const;
+	utils::duration timeOffset(unsigned int type) const;
+
+	void setRawDelay(unsigned int type, int frame, utils::duration time);
+
+	std::map<unsigned int, std::pair<int, utils::duration>> delays_;
+
+private:
+	static constexpr unsigned int HISTORY_DEPTH = 10;
+
+	void timeout(Timer *timer);
+	void updateDeadline();
+
+	std::list<std::pair<unsigned int, utils::time_point>> history_;
+	std::multimap<utils::time_point, std::unique_ptr<FrameAction>> actions_;
+	utils::duration frameInterval_;
+
+	Timer timer_;
+};
+
+} /* namespace libcamera */
+
+#endif /* __LIBCAMERA_TIMELINE_H__ */