summaryrefslogtreecommitdiff
path: root/src/android/camera_proxy.cpp
blob: 3964b5665e2f79302b46d4af58a3ae1687707c6c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
/* SPDX-License-Identifier: LGPL-2.1-or-later */
/*
 * Copyright (C) 2019, Google Inc.
 *
 * camera_proxy.cpp - Proxy to camera devices
 */

#include "camera_proxy.h"

#include <system/camera_metadata.h>

#include <libcamera/object.h>

#include "log.h"
#include "message.h"
#include "utils.h"

#include "camera_device.h"

using namespace libcamera;

LOG_DECLARE_CATEGORY(HAL);

/*
 * \class CameraProxy
 *
 * The CameraProxy wraps a CameraDevice and implements the camera3_device_t
 * API, bridging calls received from the camera framework to the CameraDevice.
 *
 * Bridging operation calls between the framework and the CameraDevice is
 * required as the two run in two different threads and certain operations,
 * such as queueing a new capture request to the camera, shall be called in
 * the thread that dispatches events. Other operations do not require any
 * bridging and resolve to direct function calls on the CameraDevice instance
 * instead.
 */

static int hal_dev_initialize(const struct camera3_device *dev,
			      const camera3_callback_ops_t *callback_ops)
{
	if (!dev)
		return -EINVAL;

	CameraProxy *proxy = reinterpret_cast<CameraProxy *>(dev->priv);
	proxy->initialize(callback_ops);

	return 0;
}

static int hal_dev_configure_streams(const struct camera3_device *dev,
				     camera3_stream_configuration_t *stream_list)
{
	if (!dev)
		return -EINVAL;

	CameraProxy *proxy = reinterpret_cast<CameraProxy *>(dev->priv);
	return proxy->configureStreams(stream_list);
}

static const camera_metadata_t *
hal_dev_construct_default_request_settings(const struct camera3_device *dev,
					   int type)
{
	if (!dev)
		return nullptr;

	CameraProxy *proxy = reinterpret_cast<CameraProxy *>(dev->priv);
	return proxy->constructDefaultRequestSettings(type);
}

static int hal_dev_process_capture_request(const struct camera3_device *dev,
					   camera3_capture_request_t *request)
{
	if (!dev)
		return -EINVAL;

	CameraProxy *proxy = reinterpret_cast<CameraProxy *>(dev->priv);
	return proxy->processCaptureRequest(request);
}

static void hal_dev_dump(const struct camera3_device *dev, int fd)
{
}

static int hal_dev_flush(const struct camera3_device *dev)
{
	return 0;
}

static int hal_dev_close(hw_device_t *hw_device)
{
	if (!hw_device)
		return -EINVAL;

	camera3_device_t *dev = reinterpret_cast<camera3_device_t *>(hw_device);
	CameraProxy *proxy = reinterpret_cast<CameraProxy *>(dev->priv);

	proxy->close();

	return 0;
}

static camera3_device_ops hal_dev_ops = {
	.initialize = hal_dev_initialize,
	.configure_streams = hal_dev_configure_streams,
	.register_stream_buffers = nullptr,
	.construct_default_request_settings = hal_dev_construct_default_request_settings,
	.process_capture_request = hal_dev_process_capture_request,
	.get_metadata_vendor_tag_ops = nullptr,
	.dump = hal_dev_dump,
	.flush = hal_dev_flush,
	.reserved = { nullptr },
};

CameraProxy::CameraProxy(unsigned int id, const std::shared_ptr<Camera> &camera)
	: id_(id)
{
	cameraDevice_ = new CameraDevice(id, camera);
}

CameraProxy::~CameraProxy()
{
	delete cameraDevice_;
}

int CameraProxy::open(const hw_module_t *hardwareModule)
{
	int ret = cameraDevice_->open();
	if (ret)
		return ret;

	/* Initialize the hw_device_t in the instance camera3_module_t. */
	camera3Device_.common.tag = HARDWARE_DEVICE_TAG;
	camera3Device_.common.version = CAMERA_DEVICE_API_VERSION_3_3;
	camera3Device_.common.module = (hw_module_t *)hardwareModule;
	camera3Device_.common.close = hal_dev_close;

	/*
	 * The camera device operations. These actually implement
	 * the Android Camera HALv3 interface.
	 */
	camera3Device_.ops = &hal_dev_ops;
	camera3Device_.priv = this;

	return 0;
}

void CameraProxy::close()
{
	cameraDevice_->invokeMethod(&CameraDevice::close,
				    ConnectionTypeBlocking);
}

void CameraProxy::initialize(const camera3_callback_ops_t *callbacks)
{
	cameraDevice_->setCallbacks(callbacks);
}

const camera_metadata_t *CameraProxy::getStaticMetadata()
{
	return cameraDevice_->getStaticMetadata();
}

const camera_metadata_t *CameraProxy::constructDefaultRequestSettings(int type)
{
	return cameraDevice_->constructDefaultRequestSettings(type);
}

int CameraProxy::configureStreams(camera3_stream_configuration_t *stream_list)
{
	return cameraDevice_->configureStreams(stream_list);
}

int CameraProxy::processCaptureRequest(camera3_capture_request_t *request)
{
	cameraDevice_->invokeMethod(&CameraDevice::processCaptureRequest,
				    ConnectionTypeBlocking, request);

	return 0;
}
'#n660'>660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294
/* SPDX-License-Identifier: LGPL-2.1-or-later */
/*
 * Copyright (C) 2019, Google Inc.
 *
 * ipu3.cpp - Pipeline handler for Intel IPU3
 */

#include <algorithm>
#include <iomanip>
#include <memory>
#include <queue>
#include <vector>

#include <linux/media-bus-format.h>

#include <libcamera/camera.h>
#include <libcamera/formats.h>
#include <libcamera/request.h>
#include <libcamera/stream.h>

#include "libcamera/internal/camera_sensor.h"
#include "libcamera/internal/device_enumerator.h"
#include "libcamera/internal/log.h"
#include "libcamera/internal/media_device.h"
#include "libcamera/internal/pipeline_handler.h"
#include "libcamera/internal/utils.h"
#include "libcamera/internal/v4l2_controls.h"
#include "libcamera/internal/v4l2_subdevice.h"
#include "libcamera/internal/v4l2_videodevice.h"

#include "cio2.h"

namespace libcamera {

LOG_DEFINE_CATEGORY(IPU3)

class ImgUDevice
{
public:
	static constexpr unsigned int PAD_INPUT = 0;
	static constexpr unsigned int PAD_OUTPUT = 2;
	static constexpr unsigned int PAD_VF = 3;
	static constexpr unsigned int PAD_STAT = 4;

	/* ImgU output descriptor: group data specific to an ImgU output. */
	struct ImgUOutput {
		V4L2VideoDevice *dev;
		unsigned int pad;
		std::string name;
	};

	ImgUDevice()
		: imgu_(nullptr), input_(nullptr)
	{
		output_.dev = nullptr;
		viewfinder_.dev = nullptr;
		stat_.dev = nullptr;
	}

	~ImgUDevice()
	{
		delete imgu_;
		delete input_;
		delete output_.dev;
		delete viewfinder_.dev;
		delete stat_.dev;
	}

	int init(MediaDevice *media, unsigned int index);
	int configureInput(const Size &size,
			   V4L2DeviceFormat *inputFormat);
	int configureOutput(ImgUOutput *output,
			    const StreamConfiguration &cfg,
			    V4L2DeviceFormat *outputFormat);

	int allocateBuffers(unsigned int bufferCount);
	void freeBuffers();

	int start();
	int stop();

	int linkSetup(const std::string &source, unsigned int sourcePad,
		      const std::string &sink, unsigned int sinkPad,
		      bool enable);
	int enableLinks(bool enable);

	unsigned int index_;
	std::string name_;
	MediaDevice *media_;

	V4L2Subdevice *imgu_;
	V4L2VideoDevice *input_;
	ImgUOutput output_;
	ImgUOutput viewfinder_;
	ImgUOutput stat_;
	/* \todo Add param video device for 3A tuning */
};

class IPU3Stream : public Stream
{
public:
	IPU3Stream()
		: active_(false), raw_(false), device_(nullptr)
	{
	}

	bool active_;
	bool raw_;
	ImgUDevice::ImgUOutput *device_;
};

class IPU3CameraData : public CameraData
{
public:
	IPU3CameraData(PipelineHandler *pipe)
		: CameraData(pipe)
	{
	}

	void imguOutputBufferReady(FrameBuffer *buffer);
	void cio2BufferReady(FrameBuffer *buffer);

	CIO2Device cio2_;
	ImgUDevice *imgu_;

	IPU3Stream outStream_;
	IPU3Stream vfStream_;
	IPU3Stream rawStream_;
};

class IPU3CameraConfiguration : public CameraConfiguration
{
public:
	IPU3CameraConfiguration(Camera *camera, IPU3CameraData *data);

	Status validate() override;

	const StreamConfiguration &cio2Format() const { return cio2Configuration_; };
	const std::vector<const IPU3Stream *> &streams() { return streams_; }

private:
	static constexpr unsigned int IPU3_BUFFER_COUNT = 4;
	static constexpr unsigned int IPU3_MAX_STREAMS = 3;

	void assignStreams();
	void adjustStream(StreamConfiguration &cfg, bool scale);

	/*
	 * The IPU3CameraData instance is guaranteed to be valid as long as the
	 * corresponding Camera instance is valid. In order to borrow a
	 * reference to the camera data, store a new reference to the camera.
	 */
	std::shared_ptr<Camera> camera_;
	const IPU3CameraData *data_;

	StreamConfiguration cio2Configuration_;
	std::vector<const IPU3Stream *> streams_;
};

class PipelineHandlerIPU3 : public PipelineHandler
{
public:
	static constexpr unsigned int V4L2_CID_IPU3_PIPE_MODE = 0x009819c1;

	enum IPU3PipeModes {
		IPU3PipeModeVideo = 0,
		IPU3PipeModeStillCapture = 1,
	};

	PipelineHandlerIPU3(CameraManager *manager);

	CameraConfiguration *generateConfiguration(Camera *camera,
		const StreamRoles &roles) override;
	int configure(Camera *camera, CameraConfiguration *config) override;

	int exportFrameBuffers(Camera *camera, Stream *stream,
			       std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;

	int start(Camera *camera) override;
	void stop(Camera *camera) override;

	int queueRequestDevice(Camera *camera, Request *request) override;

	bool match(DeviceEnumerator *enumerator) override;

private:
	IPU3CameraData *cameraData(const Camera *camera)
	{
		return static_cast<IPU3CameraData *>(
			PipelineHandler::cameraData(camera));
	}

	int registerCameras();

	int allocateBuffers(Camera *camera);
	int freeBuffers(Camera *camera);

	ImgUDevice imgu0_;
	ImgUDevice imgu1_;
	MediaDevice *cio2MediaDev_;
	MediaDevice *imguMediaDev_;
};

IPU3CameraConfiguration::IPU3CameraConfiguration(Camera *camera,
						 IPU3CameraData *data)
	: CameraConfiguration()
{
	camera_ = camera->shared_from_this();
	data_ = data;
}

void IPU3CameraConfiguration::assignStreams()
{
	/*
	 * Verify and update all configuration entries, and assign a stream to
	 * each of them. The viewfinder stream can scale, while the output
	 * stream can crop only, so select the output stream when the requested
	 * resolution is equal to the sensor resolution, and the viewfinder
	 * stream otherwise.
	 */
	std::set<const IPU3Stream *> availableStreams = {
		&data_->outStream_,
		&data_->vfStream_,
		&data_->rawStream_,
	};

	/*
	 * The caller is responsible to limit the number of requested streams
	 * to a number supported by the pipeline before calling this function.
	 */
	ASSERT(availableStreams.size() >= config_.size());

	streams_.clear();
	streams_.reserve(config_.size());

	for (const StreamConfiguration &cfg : config_) {
		const PixelFormatInfo &info =
			PixelFormatInfo::info(cfg.pixelFormat);
		const IPU3Stream *stream;

		if (info.colourEncoding == PixelFormatInfo::ColourEncodingRAW)
			stream = &data_->rawStream_;
		else if (cfg.size == cio2Configuration_.size)
			stream = &data_->outStream_;
		else
			stream = &data_->vfStream_;

		if (availableStreams.find(stream) == availableStreams.end())
			stream = *availableStreams.begin();

		streams_.push_back(stream);
		availableStreams.erase(stream);
	}
}

void IPU3CameraConfiguration::adjustStream(StreamConfiguration &cfg, bool scale)
{
	/* The only pixel format the driver supports is NV12. */
	cfg.pixelFormat = formats::NV12;

	if (scale) {
		/*
		 * Provide a suitable default that matches the sensor aspect
		 * ratio.
		 */
		if (!cfg.size.width || !cfg.size.height) {
			cfg.size.width = 1280;
			cfg.size.height = 1280 * cio2Configuration_.size.height
					/ cio2Configuration_.size.width;
		}

		/*
		 * \todo: Clamp the size to the hardware bounds when we will
		 * figure them out.
		 *
		 * \todo: Handle the scaler (BDS) restrictions. The BDS can
		 * only scale with the same factor in both directions, and the
		 * scaling factor is limited to a multiple of 1/32. At the
		 * moment the ImgU driver hides these constraints by applying
		 * additional cropping, this should be fixed on the driver
		 * side, and cropping should be exposed to us.
		 */
	} else {
		/*
		 * \todo: Properly support cropping when the ImgU driver
		 * interface will be cleaned up.
		 */
		cfg.size = cio2Configuration_.size;
	}

	/*
	 * Clamp the size to match the ImgU alignment constraints. The width
	 * shall be a multiple of 8 pixels and the height a multiple of 4
	 * pixels.
	 */
	if (cfg.size.width % 8 || cfg.size.height % 4) {
		cfg.size.width &= ~7;
		cfg.size.height &= ~3;
	}
}

CameraConfiguration::Status IPU3CameraConfiguration::validate()
{
	Status status = Valid;

	if (config_.empty())
		return Invalid;

	/* Cap the number of entries to the available streams. */
	if (config_.size() > IPU3_MAX_STREAMS) {
		config_.resize(IPU3_MAX_STREAMS);
		status = Adjusted;
	}

	/*
	 * Select the sensor format by collecting the maximum width and height
	 * and picking the closest larger match, as the IPU3 can downscale
	 * only. If no resolution is requested for any stream, or if no sensor
	 * resolution is large enough, pick the largest one.
	 */
	Size size = {};

	for (const StreamConfiguration &cfg : config_) {
		if (cfg.size.width > size.width)
			size.width = cfg.size.width;
		if (cfg.size.height > size.height)
			size.height = cfg.size.height;
	}

	/* Generate raw configuration from CIO2. */
	cio2Configuration_ = data_->cio2_.generateConfiguration(size);
	if (!cio2Configuration_.pixelFormat.isValid())
		return Invalid;

	/* Assign streams to each configuration entry. */
	assignStreams();

	/* Verify and adjust configuration if needed. */
	for (unsigned int i = 0; i < config_.size(); ++i) {
		StreamConfiguration &cfg = config_[i];
		const StreamConfiguration oldCfg = cfg;
		const IPU3Stream *stream = streams_[i];

		if (stream->raw_) {
			cfg = cio2Configuration_;
		} else {
			bool scale = stream == &data_->vfStream_;
			adjustStream(config_[i], scale);
			cfg.bufferCount = IPU3_BUFFER_COUNT;
		}

		if (cfg.pixelFormat != oldCfg.pixelFormat ||
		    cfg.size != oldCfg.size) {
			LOG(IPU3, Debug)
				<< "Stream " << i << " configuration adjusted to "
				<< cfg.toString();
			status = Adjusted;
		}
	}

	return status;
}

PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)
	: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)
{
}

CameraConfiguration *PipelineHandlerIPU3::generateConfiguration(Camera *camera,
	const StreamRoles &roles)
{
	IPU3CameraData *data = cameraData(camera);
	IPU3CameraConfiguration *config;
	std::set<IPU3Stream *> streams = {
		&data->outStream_,
		&data->vfStream_,
		&data->rawStream_,
	};

	config = new IPU3CameraConfiguration(camera, data);

	for (const StreamRole role : roles) {
		StreamConfiguration cfg = {};
		IPU3Stream *stream = nullptr;

		cfg.pixelFormat = formats::NV12;

		switch (role) {
		case StreamRole::StillCapture:
			/*
			 * Pick the output stream by default as the Viewfinder
			 * and VideoRecording roles are not allowed on
			 * the output stream.
			 */
			if (streams.find(&data->outStream_) != streams.end()) {
				stream = &data->outStream_;
			} else if (streams.find(&data->vfStream_) != streams.end()) {
				stream = &data->vfStream_;
			} else {
				LOG(IPU3, Error)
					<< "No stream available for requested role "
					<< role;
				break;
			}

			/*
			 * FIXME: Soraka: the maximum resolution reported by
			 * both sensors (2592x1944 for ov5670 and 4224x3136 for
			 * ov13858) are returned as default configurations but
			 * they're not correctly processed by the ImgU.
			 * Resolutions up tp 2560x1920 have been validated.
			 *
			 * \todo Clarify ImgU alignment requirements.
			 */
			cfg.size = { 2560, 1920 };

			break;

		case StreamRole::StillCaptureRaw: {
			if (streams.find(&data->rawStream_) == streams.end()) {
				LOG(IPU3, Error)
					<< "No stream available for requested role "
					<< role;
				break;
			}

			stream = &data->rawStream_;

			cfg.size = data->cio2_.sensor()->resolution();

			cfg = data->cio2_.generateConfiguration(cfg.size);
			break;
		}

		case StreamRole::Viewfinder:
		case StreamRole::VideoRecording: {
			/*
			 * We can't use the 'output' stream for viewfinder or
			 * video capture roles.
			 *
			 * \todo This is an artificial limitation until we
			 * figure out the exact capabilities of the hardware.
			 */
			if (streams.find(&data->vfStream_) == streams.end()) {
				LOG(IPU3, Error)
					<< "No stream available for requested role "
					<< role;
				break;
			}

			stream = &data->vfStream_;

			/*
			 * Align the default viewfinder size to the maximum
			 * available sensor resolution and to the IPU3
			 * alignment constraints.
			 */
			const Size &res = data->cio2_.sensor()->resolution();
			unsigned int width = std::min(1280U, res.width);
			unsigned int height = std::min(720U, res.height);
			cfg.size = { width & ~7, height & ~3 };

			break;
		}

		default:
			LOG(IPU3, Error)
				<< "Requested stream role not supported: " << role;
			break;
		}

		if (!stream) {
			delete config;
			return nullptr;
		}

		streams.erase(stream);

		config->addConfiguration(cfg);
	}

	config->validate();

	return config;
}

int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)
{
	IPU3CameraConfiguration *config =
		static_cast<IPU3CameraConfiguration *>(c);
	IPU3CameraData *data = cameraData(camera);
	IPU3Stream *outStream = &data->outStream_;
	IPU3Stream *vfStream = &data->vfStream_;
	CIO2Device *cio2 = &data->cio2_;
	ImgUDevice *imgu = data->imgu_;
	V4L2DeviceFormat outputFormat;
	int ret;

	/*
	 * FIXME: enabled links in one ImgU pipe interfere with capture
	 * operations on the other one. This can be easily triggered by
	 * capturing from one camera and then trying to capture from the other
	 * one right after, without disabling media links on the first used
	 * pipe.
	 *
	 * The tricky part here is where to disable links on the ImgU instance
	 * which is currently not in use:
	 * 1) Link enable/disable cannot be done at start()/stop() time as video
	 * devices needs to be linked first before format can be configured on
	 * them.
	 * 2) As link enable has to be done at the least in configure(),
	 * before configuring formats, the only place where to disable links
	 * would be 'stop()', but the Camera class state machine allows
	 * start()<->stop() sequences without any configure() in between.
	 *
	 * As of now, disable all links in the ImgU media graph before
	 * configuring the device, to allow alternate the usage of the two
	 * ImgU pipes.
	 *
	 * As a consequence, a Camera using an ImgU shall be configured before
	 * any start()/stop() sequence. An application that wants to
	 * pre-configure all the camera and then start/stop them alternatively
	 * without going through any re-configuration (a sequence that is
	 * allowed by the Camera state machine) would now fail on the IPU3.
	 */
	ret = imguMediaDev_->disableLinks();
	if (ret)
		return ret;

	/*
	 * \todo: Enable links selectively based on the requested streams.
	 * As of now, enable all links unconditionally.
	 * \todo Don't configure the ImgU at all if we only have a single
	 * stream which is for raw capture, in which case no buffers will
	 * ever be queued to the ImgU.
	 */
	ret = data->imgu_->enableLinks(true);
	if (ret)
		return ret;

	/*
	 * Pass the requested stream size to the CIO2 unit and get back the
	 * adjusted format to be propagated to the ImgU output devices.
	 */
	const Size &sensorSize = config->cio2Format().size;
	V4L2DeviceFormat cio2Format = {};
	ret = cio2->configure(sensorSize, &cio2Format);
	if (ret)
		return ret;

	ret = imgu->configureInput(sensorSize, &cio2Format);
	if (ret)
		return ret;

	/* Apply the format to the configured streams output devices. */
	outStream->active_ = false;
	vfStream->active_ = false;

	for (unsigned int i = 0; i < config->size(); ++i) {
		/*
		 * Use a const_cast<> here instead of storing a mutable stream
		 * pointer in the configuration to let the compiler catch
		 * unwanted modifications of camera data in the configuration
		 * validate() implementation.
		 */
		IPU3Stream *stream = const_cast<IPU3Stream *>(config->streams()[i]);
		StreamConfiguration &cfg = (*config)[i];

		stream->active_ = true;
		cfg.setStream(stream);

		/*
		 * The RAW still capture stream just copies buffers from the
		 * internal queue and doesn't need any specific configuration.
		 */
		if (stream->raw_) {
			cfg.stride = cio2Format.planes[0].bpl;
		} else {
			ret = imgu->configureOutput(stream->device_, cfg,
						    &outputFormat);
			if (ret)
				return ret;

			cfg.stride = outputFormat.planes[0].bpl;
		}
	}

	/*
	 * As we need to set format also on the non-active streams, use
	 * the configuration of the active one for that purpose (there should
	 * be at least one active stream in the configuration request).
	 */
	if (!outStream->active_) {
		ret = imgu->configureOutput(outStream->device_, config->at(0),
					    &outputFormat);
		if (ret)
			return ret;
	}

	if (!vfStream->active_) {
		ret = imgu->configureOutput(vfStream->device_, config->at(0),
					    &outputFormat);
		if (ret)
			return ret;
	}

	/*
	 * Apply the largest available format to the stat node.
	 * \todo Revise this when we'll actually use the stat node.
	 */
	StreamConfiguration statCfg = {};
	statCfg.size = cio2Format.size;

	ret = imgu->configureOutput(&imgu->stat_, statCfg, &outputFormat);
	if (ret)
		return ret;

	/* Apply the "pipe_mode" control to the ImgU subdevice. */
	ControlList ctrls(imgu->imgu_->controls());
	ctrls.set(V4L2_CID_IPU3_PIPE_MODE,
		  static_cast<int32_t>(vfStream->active_ ? IPU3PipeModeVideo :
				       IPU3PipeModeStillCapture));
	ret = imgu->imgu_->setControls(&ctrls);
	if (ret) {
		LOG(IPU3, Error) << "Unable to set pipe_mode control";
		return ret;
	}

	return 0;
}

int PipelineHandlerIPU3::exportFrameBuffers(Camera *camera, Stream *stream,
					    std::vector<std::unique_ptr<FrameBuffer>> *buffers)
{
	IPU3CameraData *data = cameraData(camera);
	IPU3Stream *ipu3stream = static_cast<IPU3Stream *>(stream);
	unsigned int count = stream->configuration().bufferCount;

	if (ipu3stream->raw_)
		return data->cio2_.exportBuffers(count, buffers);

	return ipu3stream->device_->dev->exportBuffers(count, buffers);
}

/**
 * \todo Clarify if 'viewfinder' and 'stat' nodes have to be set up and
 * started even if not in use. As of now, if not properly configured and
 * enabled, the ImgU processing pipeline stalls.
 *
 * In order to be able to start the 'viewfinder' and 'stat' nodes, we need
 * memory to be reserved.
 */
int PipelineHandlerIPU3::allocateBuffers(Camera *camera)
{
	IPU3CameraData *data = cameraData(camera);
	ImgUDevice *imgu = data->imgu_;
	unsigned int bufferCount;
	int ret;

	bufferCount = std::max({
		data->outStream_.configuration().bufferCount,
		data->vfStream_.configuration().bufferCount,
		data->rawStream_.configuration().bufferCount,
	});

	ret = imgu->allocateBuffers(bufferCount);
	if (ret < 0)
		return ret;

	return 0;
}

int PipelineHandlerIPU3::freeBuffers(Camera *camera)
{
	IPU3CameraData *data = cameraData(camera);

	data->imgu_->freeBuffers();

	return 0;
}

int PipelineHandlerIPU3::start(Camera *camera)
{
	IPU3CameraData *data = cameraData(camera);
	CIO2Device *cio2 = &data->cio2_;
	ImgUDevice *imgu = data->imgu_;
	int ret;

	/* Allocate buffers for internal pipeline usage. */
	ret = allocateBuffers(camera);
	if (ret)
		return ret;

	/*
	 * Start the ImgU video devices, buffers will be queued to the
	 * ImgU output and viewfinder when requests will be queued.
	 */
	ret = cio2->start();
	if (ret)
		goto error;

	ret = imgu->start();
	if (ret) {
		imgu->stop();
		cio2->stop();
		goto error;
	}

	return 0;

error:
	freeBuffers(camera);
	LOG(IPU3, Error) << "Failed to start camera " << camera->name();

	return ret;
}

void PipelineHandlerIPU3::stop(Camera *camera)
{
	IPU3CameraData *data = cameraData(camera);
	int ret = 0;

	ret |= data->imgu_->stop();
	ret |= data->cio2_.stop();
	if (ret)
		LOG(IPU3, Warning) << "Failed to stop camera "
				   << camera->name();

	freeBuffers(camera);
}

int PipelineHandlerIPU3::queueRequestDevice(Camera *camera, Request *request)
{
	IPU3CameraData *data = cameraData(camera);
	int error = 0;

	/*
	 * Queue a buffer on the CIO2, using the raw stream buffer provided in
	 * the request, if any, or a CIO2 internal buffer otherwise.
	 */
	FrameBuffer *rawBuffer = request->findBuffer(&data->rawStream_);
	error = data->cio2_.queueBuffer(request, rawBuffer);
	if (error)
		return error;

	/* Queue all buffers from the request aimed for the ImgU. */
	for (auto it : request->buffers()) {
		IPU3Stream *stream = static_cast<IPU3Stream *>(it.first);
		if (stream->raw_)
			continue;

		FrameBuffer *buffer = it.second;
		int ret = stream->device_->dev->queueBuffer(buffer);
		if (ret < 0)
			error = ret;
	}

	return error;
}

bool PipelineHandlerIPU3::match(DeviceEnumerator *enumerator)
{
	int ret;

	DeviceMatch cio2_dm("ipu3-cio2");
	cio2_dm.add("ipu3-csi2 0");
	cio2_dm.add("ipu3-cio2 0");
	cio2_dm.add("ipu3-csi2 1");
	cio2_dm.add("ipu3-cio2 1");
	cio2_dm.add("ipu3-csi2 2");
	cio2_dm.add("ipu3-cio2 2");
	cio2_dm.add("ipu3-csi2 3");
	cio2_dm.add("ipu3-cio2 3");

	DeviceMatch imgu_dm("ipu3-imgu");
	imgu_dm.add("ipu3-imgu 0");
	imgu_dm.add("ipu3-imgu 0 input");
	imgu_dm.add("ipu3-imgu 0 parameters");
	imgu_dm.add("ipu3-imgu 0 output");
	imgu_dm.add("ipu3-imgu 0 viewfinder");
	imgu_dm.add("ipu3-imgu 0 3a stat");
	imgu_dm.add("ipu3-imgu 1");
	imgu_dm.add("ipu3-imgu 1 input");
	imgu_dm.add("ipu3-imgu 1 parameters");
	imgu_dm.add("ipu3-imgu 1 output");
	imgu_dm.add("ipu3-imgu 1 viewfinder");
	imgu_dm.add("ipu3-imgu 1 3a stat");

	cio2MediaDev_ = acquireMediaDevice(enumerator, cio2_dm);
	if (!cio2MediaDev_)
		return false;

	imguMediaDev_ = acquireMediaDevice(enumerator, imgu_dm);
	if (!imguMediaDev_)
		return false;

	/*
	 * Disable all links that are enabled by default on CIO2, as camera
	 * creation enables all valid links it finds.
	 */
	if (cio2MediaDev_->disableLinks())
		return false;

	ret = imguMediaDev_->disableLinks();
	if (ret)
		return ret;

	ret = registerCameras();

	return ret == 0;
}

/**
 * \brief Initialise ImgU and CIO2 devices associated with cameras
 *
 * Initialise the two ImgU instances and create cameras with an associated
 * CIO2 device instance.
 *
 * \return 0 on success or a negative error code for error or if no camera
 * has been created
 * \retval -ENODEV no camera has been created
 */
int PipelineHandlerIPU3::registerCameras()
{
	int ret;

	ret = imgu0_.init(imguMediaDev_, 0);
	if (ret)
		return ret;

	ret = imgu1_.init(imguMediaDev_, 1);
	if (ret)
		return ret;

	/*
	 * For each CSI-2 receiver on the IPU3, create a Camera if an
	 * image sensor is connected to it and the sensor can produce images
	 * in a compatible format.
	 */
	unsigned int numCameras = 0;
	for (unsigned int id = 0; id < 4 && numCameras < 2; ++id) {
		std::unique_ptr<IPU3CameraData> data =
			std::make_unique<IPU3CameraData>(this);
		std::set<Stream *> streams = {
			&data->outStream_,
			&data->vfStream_,
			&data->rawStream_,
		};
		CIO2Device *cio2 = &data->cio2_;

		ret = cio2->init(cio2MediaDev_, id);
		if (ret)
			continue;

		/* Initialize the camera properties. */
		data->properties_ = cio2->sensor()->properties();

		/**
		 * \todo Dynamically assign ImgU and output devices to each
		 * stream and camera; as of now, limit support to two cameras
		 * only, and assign imgu0 to the first one and imgu1 to the
		 * second.
		 */
		data->imgu_ = numCameras ? &imgu1_ : &imgu0_;
		data->outStream_.device_ = &data->imgu_->output_;
		data->vfStream_.device_ = &data->imgu_->viewfinder_;
		data->rawStream_.raw_ = true;

		/*
		 * Connect video devices' 'bufferReady' signals to their
		 * slot to implement the image processing pipeline.
		 *
		 * Frames produced by the CIO2 unit are passed to the
		 * associated ImgU input where they get processed and
		 * returned through the ImgU main and secondary outputs.
		 */
		data->cio2_.bufferReady.connect(data.get(),
					&IPU3CameraData::cio2BufferReady);
		data->imgu_->input_->bufferReady.connect(&data->cio2_,
					&CIO2Device::tryReturnBuffer);
		data->imgu_->output_.dev->bufferReady.connect(data.get(),
					&IPU3CameraData::imguOutputBufferReady);
		data->imgu_->viewfinder_.dev->bufferReady.connect(data.get(),
					&IPU3CameraData::imguOutputBufferReady);

		/* Create and register the Camera instance. */
		std::string cameraName = cio2->sensor()->entity()->name();
		std::shared_ptr<Camera> camera = Camera::create(this,
								cameraName,
								streams);

		registerCamera(std::move(camera), std::move(data));

		LOG(IPU3, Info)
			<< "Registered Camera[" << numCameras << "] \""
			<< cameraName << "\""
			<< " connected to CSI-2 receiver " << id;

		numCameras++;
	}

	return numCameras ? 0 : -ENODEV;
}

/* -----------------------------------------------------------------------------
 * Buffer Ready slots
 */

/**
 * \brief Handle buffers completion at the ImgU output
 * \param[in] buffer The completed buffer
 *
 * Buffers completed from the ImgU output are directed to the application.
 */
void IPU3CameraData::imguOutputBufferReady(FrameBuffer *buffer)
{
	Request *request = buffer->request();

	if (!pipe_->completeBuffer(camera_, request, buffer))
		/* Request not completed yet, return here. */
		return;

	/* Mark the request as complete. */
	pipe_->completeRequest(camera_, request);
}

/**
 * \brief Handle buffers completion at the CIO2 output
 * \param[in] buffer The completed buffer
 *
 * Buffers completed from the CIO2 are immediately queued to the ImgU unit
 * for further processing.
 */
void IPU3CameraData::cio2BufferReady(FrameBuffer *buffer)
{
	/* \todo Handle buffer failures when state is set to BufferError. */
	if (buffer->metadata().status == FrameMetadata::FrameCancelled)
		return;

	Request *request = buffer->request();

	/*
	 * If the request contains a buffer for the RAW stream only, complete it
	 * now as there's no need for ImgU processing.
	 */
	if (request->findBuffer(&rawStream_) &&
	    pipe_->completeBuffer(camera_, request, buffer)) {
		pipe_->completeRequest(camera_, request);
		return;
	}

	imgu_->input_->queueBuffer(buffer);
}

/* -----------------------------------------------------------------------------
 * ImgU Device
 */

/**
 * \brief Initialize components of the ImgU instance
 * \param[in] mediaDevice The ImgU instance media device
 * \param[in] index The ImgU instance index
 *
 * Create and open the V4L2 devices and subdevices of the ImgU instance
 * with \a index.
 *
 * In case of errors the created V4L2VideoDevice and V4L2Subdevice instances
 * are destroyed at pipeline handler delete time.
 *
 * \return 0 on success or a negative error code otherwise
 */
int ImgUDevice::init(MediaDevice *media, unsigned int index)
{
	int ret;

	index_ = index;
	name_ = "ipu3-imgu " + std::to_string(index_);
	media_ = media;

	/*
	 * The media entities presence in the media device has been verified
	 * by the match() function: no need to check for newly created
	 * video devices and subdevice validity here.
	 */
	imgu_ = V4L2Subdevice::fromEntityName(media, name_);
	ret = imgu_->open();
	if (ret)
		return ret;

	input_ = V4L2VideoDevice::fromEntityName(media, name_ + " input");
	ret = input_->open();
	if (ret)
		return ret;

	output_.dev = V4L2VideoDevice::fromEntityName(media, name_ + " output");
	ret = output_.dev->open();
	if (ret)
		return ret;

	output_.pad = PAD_OUTPUT;
	output_.name = "output";

	viewfinder_.dev = V4L2VideoDevice::fromEntityName(media,
							  name_ + " viewfinder");
	ret = viewfinder_.dev->open();
	if (ret)
		return ret;

	viewfinder_.pad = PAD_VF;
	viewfinder_.name = "viewfinder";

	stat_.dev = V4L2VideoDevice::fromEntityName(media, name_ + " 3a stat");
	ret = stat_.dev->open();
	if (ret)
		return ret;

	stat_.pad = PAD_STAT;
	stat_.name = "stat";

	return 0;
}

/**
 * \brief Configure the ImgU unit input
 * \param[in] size The ImgU input frame size
 * \param[in] inputFormat The format to be applied to ImgU input
 * \return 0 on success or a negative error code otherwise
 */
int ImgUDevice::configureInput(const Size &size,
			       V4L2DeviceFormat *inputFormat)
{
	/* Configure the ImgU input video device with the requested sizes. */
	int ret = input_->setFormat(inputFormat);
	if (ret)
		return ret;

	LOG(IPU3, Debug) << "ImgU input format = " << inputFormat->toString();

	/*
	 * \todo The IPU3 driver implementation shall be changed to use the
	 * input sizes as 'ImgU Input' subdevice sizes, and use the desired
	 * GDC output sizes to configure the crop/compose rectangles.
	 *
	 * The current IPU3 driver implementation uses GDC sizes as the
	 * 'ImgU Input' subdevice sizes, and the input video device sizes
	 * to configure the crop/compose rectangles, contradicting the
	 * V4L2 specification.
	 */
	Rectangle rect = {
		.x = 0,
		.y = 0,
		.width = inputFormat->size.width,
		.height = inputFormat->size.height,
	};
	ret = imgu_->setSelection(PAD_INPUT, V4L2_SEL_TGT_CROP, &rect);
	if (ret)
		return ret;

	ret = imgu_->setSelection(PAD_INPUT, V4L2_SEL_TGT_COMPOSE, &rect);
	if (ret)
		return ret;

	LOG(IPU3, Debug) << "ImgU input feeder and BDS rectangle = "
			 << rect.toString();

	V4L2SubdeviceFormat imguFormat = {};
	imguFormat.mbus_code = MEDIA_BUS_FMT_FIXED;
	imguFormat.size = size;

	ret = imgu_->setFormat(PAD_INPUT, &imguFormat);
	if (ret)
		return ret;

	LOG(IPU3, Debug) << "ImgU GDC format = " << imguFormat.toString();

	return 0;
}

/**
 * \brief Configure the ImgU unit \a id video output
 * \param[in] output The ImgU output device to configure
 * \param[in] cfg The requested configuration
 * \return 0 on success or a negative error code otherwise
 */
int ImgUDevice::configureOutput(ImgUOutput *output,
				const StreamConfiguration &cfg,
				V4L2DeviceFormat *outputFormat)
{
	V4L2VideoDevice *dev = output->dev;
	unsigned int pad = output->pad;

	V4L2SubdeviceFormat imguFormat = {};
	imguFormat.mbus_code = MEDIA_BUS_FMT_FIXED;
	imguFormat.size = cfg.size;

	int ret = imgu_->setFormat(pad, &imguFormat);
	if (ret)
		return ret;

	/* No need to apply format to the stat node. */
	if (output == &stat_)
		return 0;

	*outputFormat = {};
	outputFormat->fourcc = dev->toV4L2PixelFormat(formats::NV12);
	outputFormat->size = cfg.size;
	outputFormat->planesCount = 2;

	ret = dev->setFormat(outputFormat);
	if (ret)
		return ret;

	LOG(IPU3, Debug) << "ImgU " << output->name << " format = "
			 << outputFormat->toString();

	return 0;
}

/**
 * \brief Allocate buffers for all the ImgU video devices
 */
int ImgUDevice::allocateBuffers(unsigned int bufferCount)
{
	/* Share buffers between CIO2 output and ImgU input. */
	int ret = input_->importBuffers(bufferCount);
	if (ret) {
		LOG(IPU3, Error) << "Failed to import ImgU input buffers";
		return ret;
	}

	/*
	 * The kernel fails to start if buffers are not either imported or
	 * allocated for the statistics video device. As statistics buffers are
	 * not yet used by the pipeline import buffers to save memory.
	 *
	 * \todo To be revised when we'll actually use the stat node.
	 */
	ret = stat_.dev->importBuffers(bufferCount);
	if (ret < 0) {
		LOG(IPU3, Error) << "Failed to allocate ImgU stat buffers";
		goto error;
	}

	/*
	 * Import buffers for all outputs, regardless of whether the
	 * corresponding stream is active or inactive, as the driver needs
	 * buffers to be requested on the V4L2 devices in order to operate.
	 */
	ret = output_.dev->importBuffers(bufferCount);
	if (ret < 0) {
		LOG(IPU3, Error) << "Failed to import ImgU output buffers";
		goto error;
	}

	ret = viewfinder_.dev->importBuffers(bufferCount);
	if (ret < 0) {
		LOG(IPU3, Error) << "Failed to import ImgU viewfinder buffers";
		goto error;
	}

	return 0;

error:
	freeBuffers();

	return ret;
}

/**
 * \brief Release buffers for all the ImgU video devices
 */
void ImgUDevice::freeBuffers()
{
	int ret;

	ret = output_.dev->releaseBuffers();
	if (ret)
		LOG(IPU3, Error) << "Failed to release ImgU output buffers";

	ret = stat_.dev->releaseBuffers();
	if (ret)
		LOG(IPU3, Error) << "Failed to release ImgU stat buffers";

	ret = viewfinder_.dev->releaseBuffers();
	if (ret)
		LOG(IPU3, Error) << "Failed to release ImgU viewfinder buffers";

	ret = input_->releaseBuffers();
	if (ret)
		LOG(IPU3, Error) << "Failed to release ImgU input buffers";
}

int ImgUDevice::start()
{
	int ret;

	/* Start the ImgU video devices. */
	ret = output_.dev->streamOn();
	if (ret) {
		LOG(IPU3, Error) << "Failed to start ImgU output";
		return ret;
	}

	ret = viewfinder_.dev->streamOn();
	if (ret) {
		LOG(IPU3, Error) << "Failed to start ImgU viewfinder";
		return ret;
	}

	ret = stat_.dev->streamOn();
	if (ret) {
		LOG(IPU3, Error) << "Failed to start ImgU stat";
		return ret;
	}

	ret = input_->streamOn();
	if (ret) {
		LOG(IPU3, Error) << "Failed to start ImgU input";
		return ret;
	}

	return 0;
}

int ImgUDevice::stop()
{
	int ret;

	ret = output_.dev->streamOff();
	ret |= viewfinder_.dev->streamOff();
	ret |= stat_.dev->streamOff();
	ret |= input_->streamOff();

	return ret;
}

/**
 * \brief Enable or disable a single link on the ImgU instance
 *
 * This method assumes the media device associated with the ImgU instance
 * is open.
 *
 * \return 0 on success or a negative error code otherwise
 */
int ImgUDevice::linkSetup(const std::string &source, unsigned int sourcePad,
			  const std::string &sink, unsigned int sinkPad,
			  bool enable)
{
	MediaLink *link = media_->link(source, sourcePad, sink, sinkPad);
	if (!link) {
		LOG(IPU3, Error)
			<< "Failed to get link: '" << source << "':"
			<< sourcePad << " -> '" << sink << "':" << sinkPad;
		return -ENODEV;
	}

	return link->setEnabled(enable);
}

/**
 * \brief Enable or disable all media links in the ImgU instance to prepare
 * for capture operations
 *
 * \todo This method will probably be removed or changed once links will be
 * enabled or disabled selectively.
 *
 * \return 0 on success or a negative error code otherwise
 */
int ImgUDevice::enableLinks(bool enable)
{
	std::string viewfinderName = name_ + " viewfinder";
	std::string outputName = name_ + " output";
	std::string statName = name_ + " 3a stat";
	std::string inputName = name_ + " input";
	int ret;

	ret = linkSetup(inputName, 0, name_, PAD_INPUT, enable);
	if (ret)
		return ret;

	ret = linkSetup(name_, PAD_OUTPUT, outputName, 0, enable);
	if (ret)
		return ret;

	ret = linkSetup(name_, PAD_VF, viewfinderName, 0, enable);
	if (ret)
		return ret;

	return linkSetup(name_, PAD_STAT, statName, 0, enable);
}

REGISTER_PIPELINE_HANDLER(PipelineHandlerIPU3);

} /* namespace libcamera */