src/ipa/raspberrypi/cam_helper.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

/* SPDX-License-Identifier: BSD-2-Clause */
/*
 * Copyright (C) 2019, Raspberry Pi Ltd
 *
 * cam_helper.h - helper class providing camera information
 */
#pragma once

#include <memory>
#include <string>
#include <utility>

#include <libcamera/base/span.h>
#include <libcamera/base/utils.h>

#include "camera_mode.h"
#include "controller/controller.h"
#include "controller/metadata.h"
#include "md_parser.h"

#include "libcamera/internal/v4l2_videodevice.h"

namespace RPiController {

/*
 * The CamHelper class provides a number of facilities that anyone trying
 * to drive a camera will need to know, but which are not provided by the
 * standard driver framework. Specifically, it provides:
 *
 * A "CameraMode" structure to describe extra information about the chosen
 * mode of the driver. For example, how it is cropped from the full sensor
 * area, how it is scaled, whether pixels are averaged compared to the full
 * resolution.
 *
 * The ability to convert between number of lines of exposure and actual
 * exposure time, and to convert between the sensor's gain codes and actual
 * gains.
 *
 * A function to return the number of frames of delay between updating exposure,
 * analogue gain and vblanking, and for the changes to take effect. For many
 * sensors these take the values 2, 1 and 2 respectively, but sensors that are
 * different will need to over-ride the default function provided.
 *
 * A function to query if the sensor outputs embedded data that can be parsed.
 *
 * A function to return the sensitivity of a given camera mode.
 *
 * A parser to parse the embedded data buffers provided by some sensors (for
 * example, the imx219 does; the ov5647 doesn't). This allows us to know for
 * sure the exposure and gain of the frame we're looking at. CamHelper
 * provides functions for converting analogue gains to and from the sensor's
 * native gain codes.
 *
 * Finally, a set of functions that determine how to handle the vagaries of
 * different camera modules on start-up or when switching modes. Some
 * modules may produce one or more frames that are not yet correctly exposed,
 * or where the metadata may be suspect. We have the following functions:
 * HideFramesStartup(): Tell the pipeline handler not to return this many
 *     frames at start-up. This can also be used to hide initial frames
 *     while the AGC and other algorithms are sorting themselves out.
 * HideFramesModeSwitch(): Tell the pipeline handler not to return this
 *     many frames after a mode switch (other than start-up). Some sensors
 *     may produce innvalid frames after a mode switch; others may not.
 * MistrustFramesStartup(): At start-up a sensor may return frames for
 *    which we should not run any control algorithms (for example, metadata
 *    may be invalid).
 * MistrustFramesModeSwitch(): The number of frames, after a mode switch
 *    (other than start-up), for which control algorithms should not run
 *    (for example, metadata may be unreliable).
 */

class CamHelper
{
public:
	static CamHelper *create(std::string const &camName);
	CamHelper(std::unique_ptr<MdParser> parser, unsigned int frameIntegrationDiff);
	virtual ~CamHelper();
	void setCameraMode(const CameraMode &mode);
	virtual void prepare(libcamera::Span<const uint8_t> buffer,
			     Metadata &metadata);
	virtual void process(StatisticsPtr &stats, Metadata &metadata);
	virtual uint32_t exposureLines(const libcamera::utils::Duration exposure,
				       const libcamera::utils::Duration lineLength) const;
	virtual libcamera::utils::Duration exposure(uint32_t exposureLines,
						    const libcamera::utils::Duration lineLength) const;
	virtual std::pair<uint32_t, uint32_t> getBlanking(libcamera::utils::Duration &exposure,
							  libcamera::utils::Duration minFrameDuration,
							  libcamera::utils::Duration maxFrameDuration) const;
	libcamera::utils::Duration hblankToLineLength(uint32_t hblank) const;
	uint32_t lineLengthToHblank(const libcamera::utils::Duration &duration) const;
	libcamera::utils::Duration lineLengthPckToDuration(uint32_t lineLengthPck) const;
	virtual uint32_t gainCode(double gain) const = 0;
	virtual double gain(uint32_t gainCode) const = 0;
	virtual void getDelays(int &exposureDelay, int &gainDelay,
			       int &vblankDelay, int &hblankDelay) const;
	virtual bool sensorEmbeddedDataPresent() const;
	virtual double getModeSensitivity(const CameraMode &mode) const;
	virtual unsigned int hideFramesStartup() const;
	virtual unsigned int hideFramesModeSwitch() const;
	virtual unsigned int mistrustFramesStartup() const;
	virtual unsigned int mistrustFramesModeSwitch() const;

protected:
	void parseEmbeddedData(libcamera::Span<const uint8_t> buffer,
			       Metadata &metadata);
	virtual void populateMetadata(const MdParser::RegisterMap &registers,
				      Metadata &metadata) const;

	std::unique_ptr<MdParser> parser_;
	CameraMode mode_;

private:
	/*
	 * Smallest difference between the frame length and integration time,
	 * in units of lines.
	 */
	unsigned int frameIntegrationDiff_;
};

/*
 * This is for registering camera helpers with the system, so that the
 * CamHelper::Create function picks them up automatically.
 */

typedef CamHelper *(*CamHelperCreateFunc)();
struct RegisterCamHelper
{
	RegisterCamHelper(char const *camName,
			  CamHelperCreateFunc createFunc);
};

} /* namespace RPi */
/* SPDX-License-Identifier: LGPL-2.1-or-later */
/*
 * Copyright (C) 2019, Google Inc.
 *
 * rkisp1.cpp - Pipeline handler for Rockchip ISP1
 */

#include <algorithm>
#include <array>
#include <iomanip>
#include <memory>
#include <numeric>
#include <queue>

#include <linux/media-bus-format.h>

#include <libcamera/buffer.h>
#include <libcamera/camera.h>
#include <libcamera/control_ids.h>
#include <libcamera/formats.h>
#include <libcamera/ipa/rkisp1.h>
#include <libcamera/request.h>
#include <libcamera/stream.h>

#include "libcamera/internal/camera_sensor.h"
#include "libcamera/internal/device_enumerator.h"
#include "libcamera/internal/ipa_manager.h"
#include "libcamera/internal/log.h"
#include "libcamera/internal/media_device.h"
#include "libcamera/internal/pipeline_handler.h"
#include "libcamera/internal/utils.h"
#include "libcamera/internal/v4l2_subdevice.h"
#include "libcamera/internal/v4l2_videodevice.h"

#include "rkisp1_path.h"
#include "timeline.h"

namespace libcamera {

LOG_DEFINE_CATEGORY(RkISP1)

class PipelineHandlerRkISP1;
class RkISP1ActionQueueBuffers;
class RkISP1CameraData;

enum RkISP1ActionType {
	SetSensor,
	SOE,
	QueueBuffers,
};

struct RkISP1FrameInfo {
	unsigned int frame;
	Request *request;

	FrameBuffer *paramBuffer;
	FrameBuffer *statBuffer;
	FrameBuffer *mainPathBuffer;
	FrameBuffer *selfPathBuffer;

	bool paramFilled;
	bool paramDequeued;
	bool metadataProcessed;
};

class RkISP1Frames
{
public:
	RkISP1Frames(PipelineHandler *pipe);

	RkISP1FrameInfo *create(const RkISP1CameraData *data, Request *request);
	int destroy(unsigned int frame);
	void clear();

	RkISP1FrameInfo *find(unsigned int frame);
	RkISP1FrameInfo *find(FrameBuffer *buffer);
	RkISP1FrameInfo *find(Request *request);

private:
	PipelineHandlerRkISP1 *pipe_;
	std::map<unsigned int, RkISP1FrameInfo *> frameInfo_;
};

class RkISP1Timeline : public Timeline
{
public:
	RkISP1Timeline()
		: Timeline()
	{
		setDelay(SetSensor, -1, 5);
		setDelay(SOE, 0, -1);
		setDelay(QueueBuffers, -1, 10);
	}

	void bufferReady(FrameBuffer *buffer)
	{
		/*
		 * Calculate SOE by taking the end of DMA set by the kernel and applying
		 * the time offsets provideprovided by the IPA to find the best estimate
		 * of SOE.
		 */

		ASSERT(frameOffset(SOE) == 0);

		utils::time_point soe = std::chrono::time_point<utils::clock>()
			+ std::chrono::nanoseconds(buffer->metadata().timestamp)
			+ timeOffset(SOE);

		notifyStartOfExposure(buffer->metadata().sequence, soe);
	}

	void setDelay(unsigned int type, int frame, int msdelay)
	{
		utils::duration delay = std::chrono::milliseconds(msdelay);
		setRawDelay(type, frame, delay);
	}
};

class RkISP1CameraData : public CameraData
{
public:
	RkISP1CameraData(PipelineHandler *pipe, RkISP1MainPath *mainPath,
			 RkISP1SelfPath *selfPath)
		: CameraData(pipe), frame_(0), frameInfo_(pipe),
		  mainPath_(mainPath), selfPath_(selfPath)
	{
	}

	int loadIPA();

	Stream mainPathStream_;
	Stream selfPathStream_;
	std::unique_ptr<CameraSensor> sensor_;
	unsigned int frame_;
	std::vector<IPABuffer> ipaBuffers_;
	RkISP1Frames frameInfo_;
	RkISP1Timeline timeline_;

	RkISP1MainPath *mainPath_;
	RkISP1SelfPath *selfPath_;

private:
	void queueFrameAction(unsigned int frame,
			      const IPAOperationData &action);

	void metadataReady(unsigned int frame, const ControlList &metadata);
};

class RkISP1CameraConfiguration : public CameraConfiguration
{
public:
	RkISP1CameraConfiguration(Camera *camera, RkISP1CameraData *data);

	Status validate() override;

	const V4L2SubdeviceFormat &sensorFormat() { return sensorFormat_; }

private:
	bool fitsAllPaths(const StreamConfiguration &cfg);

	/*
	 * The RkISP1CameraData instance is guaranteed to be valid as long as the
	 * corresponding Camera instance is valid. In order to borrow a
	 * reference to the camera data, store a new reference to the camera.
	 */
	std::shared_ptr<Camera> camera_;
	const RkISP1CameraData *data_;

	V4L2SubdeviceFormat sensorFormat_;
};

class PipelineHandlerRkISP1 : public PipelineHandler
{
public:
	PipelineHandlerRkISP1(CameraManager *manager);

	CameraConfiguration *generateConfiguration(Camera *camera,
		const StreamRoles &roles) override;
	int configure(Camera *camera, CameraConfiguration *config) override;

	int exportFrameBuffers(Camera *camera, Stream *stream,
			       std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;

	int start(Camera *camera, ControlList *controls) override;
	void stop(Camera *camera) override;

	int queueRequestDevice(Camera *camera, Request *request) override;

	bool match(DeviceEnumerator *enumerator) override;

private:
	RkISP1CameraData *cameraData(const Camera *camera)
	{
		return static_cast<RkISP1CameraData *>(
			PipelineHandler::cameraData(camera));
	}

	friend RkISP1ActionQueueBuffers;
	friend RkISP1CameraData;
	friend RkISP1Frames;

	int initLinks(const Camera *camera, const CameraSensor *sensor,
		      const RkISP1CameraConfiguration &config);
	int createCamera(MediaEntity *sensor);
	void tryCompleteRequest(Request *request);
	void bufferReady(FrameBuffer *buffer);
	void paramReady(FrameBuffer *buffer);
	void statReady(FrameBuffer *buffer);

	int allocateBuffers(Camera *camera);
	int freeBuffers(Camera *camera);

	MediaDevice *media_;
	std::unique_ptr<V4L2Subdevice> isp_;
	std::unique_ptr<V4L2VideoDevice> param_;
	std::unique_ptr<V4L2VideoDevice> stat_;

	RkISP1MainPath mainPath_;
	RkISP1SelfPath selfPath_;

	std::vector<std::unique_ptr<FrameBuffer>> paramBuffers_;
	std::vector<std::unique_ptr<FrameBuffer>> statBuffers_;
	std::queue<FrameBuffer *> availableParamBuffers_;
	std::queue<FrameBuffer *> availableStatBuffers_;

	Camera *activeCamera_;
};

RkISP1Frames::RkISP1Frames(PipelineHandler *pipe)
	: pipe_(static_cast<PipelineHandlerRkISP1 *>(pipe))
{
}

RkISP1FrameInfo *RkISP1Frames::create(const RkISP1CameraData *data, Request *request)
{
	unsigned int frame = data->frame_;

	if (pipe_->availableParamBuffers_.empty()) {
		LOG(RkISP1, Error) << "Parameters buffer underrun";
		return nullptr;
	}
	FrameBuffer *paramBuffer = pipe_->availableParamBuffers_.front();

	if (pipe_->availableStatBuffers_.empty()) {
		LOG(RkISP1, Error) << "Statisitc buffer underrun";
		return nullptr;
	}
	FrameBuffer *statBuffer = pipe_->availableStatBuffers_.front();

	FrameBuffer *mainPathBuffer = request->findBuffer(&data->mainPathStream_);
	FrameBuffer *selfPathBuffer = request->findBuffer(&data->selfPathStream_);

	pipe_->availableParamBuffers_.pop();
	pipe_->availableStatBuffers_.pop();

	RkISP1FrameInfo *info = new RkISP1FrameInfo;

	info->frame = frame;
	info->request = request;
	info->paramBuffer = paramBuffer;
	info->mainPathBuffer = mainPathBuffer;
	info->selfPathBuffer = selfPathBuffer;
	info->statBuffer = statBuffer;
	info->paramFilled = false;
	info->paramDequeued = false;
	info->metadataProcessed = false;

	frameInfo_[frame] = info;

	return info;
}

int RkISP1Frames::destroy(unsigned int frame)
{
	RkISP1FrameInfo *info = find(frame);
	if (!info)
		return -ENOENT;

	pipe_->availableParamBuffers_.push(info->paramBuffer);
	pipe_->availableStatBuffers_.push(info->statBuffer);

	frameInfo_.erase(info->frame);

	delete info;

	return 0;
}

void RkISP1Frames::clear()
{
	for (const auto &entry : frameInfo_) {
		RkISP1FrameInfo *info = entry.second;

		pipe_->availableParamBuffers_.push(info->paramBuffer);
		pipe_->availableStatBuffers_.push(info->statBuffer);

		delete info;
	}

	frameInfo_.clear();
}

RkISP1FrameInfo *RkISP1Frames::find(unsigned int frame)
{