src/android/camera_hal_manager.cpp

/* SPDX-License-Identifier: LGPL-2.1-or-later */
/*
 * Copyright (C) 2019, Google Inc.
 *
 * ipu3.cpp - Pipeline handler for Intel IPU3
 */

#include <algorithm>
#include <iomanip>
#include <memory>
#include <queue>
#include <vector>

#include <libcamera/camera.h>
#include <libcamera/control_ids.h>
#include <libcamera/formats.h>
#include <libcamera/ipa/ipu3.h>
#include <libcamera/request.h>
#include <libcamera/stream.h>

#include "libcamera/internal/camera_sensor.h"
#include "libcamera/internal/delayed_controls.h"
#include "libcamera/internal/device_enumerator.h"
#include "libcamera/internal/ipa_manager.h"
#include "libcamera/internal/log.h"
#include "libcamera/internal/media_device.h"
#include "libcamera/internal/pipeline_handler.h"
#include "libcamera/internal/utils.h"
#include "libcamera/internal/v4l2_controls.h"

#include "cio2.h"
#include "frames.h"
#include "imgu.h"

namespace libcamera {

LOG_DEFINE_CATEGORY(IPU3)

static constexpr unsigned int IPU3_BUFFER_COUNT = 4;
static constexpr unsigned int IPU3_MAX_STREAMS = 3;
static const Size IMGU_OUTPUT_MIN_SIZE = { 2, 2 };
static const Size IMGU_OUTPUT_MAX_SIZE = { 4480, 34004 };
static constexpr unsigned int IMGU_OUTPUT_WIDTH_ALIGN = 64;
static constexpr unsigned int IMGU_OUTPUT_HEIGHT_ALIGN = 4;
static constexpr unsigned int IMGU_OUTPUT_WIDTH_MARGIN = 64;
static constexpr unsigned int IMGU_OUTPUT_HEIGHT_MARGIN = 32;
static constexpr Size IPU3ViewfinderSize(1280, 720);

static const ControlInfoMap::Map IPU3Controls = {
	{ &controls::draft::PipelineDepth, ControlInfo(2, 3) },
};

class IPU3CameraData : public CameraData
{
public:
	IPU3CameraData(PipelineHandler *pipe)
		: CameraData(pipe), exposureTime_(0)
	{
	}

	int loadIPA();

	void imguOutputBufferReady(FrameBuffer *buffer);
	void cio2BufferReady(FrameBuffer *buffer);
	void paramBufferReady(FrameBuffer *buffer);
	void statBufferReady(FrameBuffer *buffer);

	CIO2Device cio2_;
	ImgUDevice *imgu_;

	Stream outStream_;
	Stream vfStream_;
	Stream rawStream_;

	uint32_t exposureTime_;
	Rectangle cropRegion_;
	std::unique_ptr<DelayedControls> delayedCtrls_;
	IPU3Frames frameInfos_;

private:
	void queueFrameAction(unsigned int id, const IPAOperationData &op);
};

class IPU3CameraConfiguration : public CameraConfiguration
{
public:
	IPU3CameraConfiguration(IPU3CameraData *data);

	Status validate() override;

	const StreamConfiguration &cio2Format() const { return cio2Configuration_; }
	const ImgUDevice::PipeConfig imguConfig() const { return pipeConfig_; }

private:
	/*
	 * The IPU3CameraData instance is guaranteed to be valid as long as the
	 * corresponding Camera instance is valid. In order to borrow a
	 * reference to the camera data, store a new reference to the camera.
	 */
	const IPU3CameraData *data_;

	StreamConfiguration cio2Configuration_;
	ImgUDevice::PipeConfig pipeConfig_;
};

class PipelineHandlerIPU3 : public PipelineHandler
{
public:
	static constexpr unsigned int V4L2_CID_IPU3_PIPE_MODE = 0x009819c1;

	enum IPU3PipeModes {
		IPU3PipeModeVideo = 0,
		IPU3PipeModeStillCapture = 1,
	};

	PipelineHandlerIPU3(CameraManager *manager);

	CameraConfiguration *generateConfiguration(Camera *camera,
		const StreamRoles &roles) override;
	int configure(Camera *camera, CameraConfiguration *config) override;

	int exportFrameBuffers(Camera *camera, Stream *stream,
			       std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;

	int start(Camera *camera, ControlList *controls) override;
	void stop(Camera *camera) override;

	int queueRequestDevice(Camera *camera, Request *request) override;

	bool match(DeviceEnumerator *enumerator) override;

private:
	IPU3CameraData *cameraData(const Camera *camera)
	{
		return static_cast<IPU3CameraData *>(
			PipelineHandler::cameraData(camera));
	}

	int initControls(IPU3CameraData *data);
	int registerCameras();

	int allocateBuffers(Camera *camera);
	int freeBuffers(Camera *camera);

	ImgUDevice imgu0_;
	ImgUDevice imgu1_;
	MediaDevice *cio2MediaDev_;
	MediaDevice *imguMediaDev_;

	std::vector<IPABuffer> ipaBuffers_;
};

IPU3CameraConfiguration::IPU3CameraConfiguration(IPU3CameraData *data)
	: CameraConfiguration()
{
	data_ = data;
}

CameraConfiguration::Status IPU3CameraConfiguration::validate()
{
	Status status = Valid;

	if (config_.empty())
		return Invalid;

	if (transform != Transform::Identity) {
		transform = Transform::Identity;
		status = Adjusted;
	}

	/* Cap the number of entries to the available streams. */
	if (config_.size() > IPU3_MAX_STREAMS) {
		config_.resize(IPU3_MAX_STREAMS);
		status = Adjusted;
	}

	/* Validate the requested stream configuration */
	unsigned int rawCount = 0;
	unsigned int yuvCount = 0;
	Size maxYuvSize;
	Size rawSize;

	for (const StreamConfiguration &cfg : config_) {
		const PixelFormatInfo &info = PixelFormatInfo::info(cfg.pixelFormat);

		if (info.colourEncoding == PixelFormatInfo::ColourEncodingRAW) {
			rawCount++;
			rawSize = cfg.size;
		} else {
			yuvCount++;
			maxYuvSize.expandTo(cfg.size);
		}
	}

	if (rawCount > 1 || yuvCount > 2) {
		LOG(IPU3, Debug) << "Camera configuration not supported";
		return Invalid;
	}

	/*
	 * Generate raw configuration from CIO2.
	 *
	 * \todo The image sensor frame size should be selected to optimize
	 * operations based on the sizes of the requested streams. However such
	 * a selection makes the pipeline configuration procedure fail for small
	 * resolutions (for example: 640x480 with OV5670) and causes the capture
	 * operations to stall for some stream size combinations (see the
	 * commit message of the patch that introduced this comment for more
	 * failure examples).
	 *
	 * Until the sensor frame size calculation criteria are clarified, when
	 * capturing from ImgU always use the largest possible size which
	 * guarantees better results at the expense of the frame rate and CSI-2
	 * bus bandwidth. When only a raw stream is requested use the requested
	 * size instead, as the ImgU is not involved.
	 */
	if (!yuvCount)
		cio2Configuration_ = data_->cio2_.generateConfiguration(rawSize);
	else
		cio2Configuration_ = data_->cio2_.generateConfiguration({});
	if (!cio2Configuration_.pixelFormat.isValid())
		return Invalid;

	LOG(IPU3, Debug) << "CIO2 configuration: " << cio2Configuration_.toString();

	ImgUDevice::Pipe pipe{};
	pipe.input = cio2Configuration_.size;

	/*
	 * Adjust the configurations if needed and assign streams while
	 * iterating them.
	 */
	bool mainOutputAvailable = true;
	for (unsigned int i = 0; i < config_.size(); ++i) {
		const PixelFormatInfo &info = PixelFormatInfo::info(config_[i].pixelFormat);
		const StreamConfiguration originalCfg = config_[i];
		StreamConfiguration *cfg = &config_[i];

		LOG(IPU3, Debug) << "Validating stream: " << config_[i].toString();

		if (info.colourEncoding == PixelFormatInfo::ColourEncodingRAW) {
			/* Initialize the RAW stream with the CIO2 configuration. */
			cfg->size = cio2Configuration_.size;
			cfg->pixelFormat = cio2Configuration_.pixelFormat;
			cfg->bufferCount = cio2Configuration_.bufferCount;
			cfg->stride = info.stride(cfg->size.width, 0, 64);
			cfg->frameSize = info.frameSize(cfg->size, 64);
			cfg->setStream(const_cast<Stream *>(&data_->rawStream_));

			LOG(IPU3, Debug) << "Assigned " << cfg->toString()
					 << " to the raw stream";
		} else {
			/* Assign and configure the main and viewfinder outputs. */

			/*
			 * Clamp the size to match the ImgU size limits and the
			 * margins from the CIO2 output frame size.
			 *
			 * The ImgU outputs needs to be strictly smaller than
			 * the CIO2 output frame and rounded down to 64 pixels
			 * in width and 32 pixels in height. This assumption
			 * comes from inspecting the pipe configuration script
			 * results and the available suggested configurations in
			 * the ChromeOS BSP .xml camera tuning files and shall
			 * be validated.
			 *
			 * \todo Clarify what are the hardware constraints
			 * that require this alignements, if any. It might
			 * depend on the BDS scaling factor of 1/32, as the main
			 * output has no YUV scaler as the viewfinder output has.
			 */
			unsigned int limit;
			limit = utils::alignDown(cio2Configuration_.size.width - 1,
						 IMGU_OUTPUT_WIDTH_MARGIN);
			cfg->size.width = std::clamp(cfg->size.width,
						     IMGU_OUTPUT_MIN_SIZE.width,
						     limit);

			limit = utils::alignDown(cio2Configuration_.size.height - 1,
						 IMGU_OUTPUT_HEIGHT_MARGIN);
			cfg->size.height = std::clamp(cfg->size.height,
						      IMGU_OUTPUT_MIN_SIZE.height,
						      limit);

			cfg->size.alignDownTo(IMGU_OUTPUT_WIDTH_ALIGN,
					      IMGU_OUTPUT_HEIGHT_ALIGN);

			cfg->pixelFormat = formats::NV12;
			cfg->bufferCount = IPU3_BUFFER_COUNT;
			cfg->stride = info.stride(cfg->size.width, 0, 1);
			cfg->frameSize = info.frameSize(cfg->size, 1);

			/*
			 * Use the main output stream in case only one stream is
			 * requested or if the current configuration is the one
			 * with the maximum YUV output size.
			 */
			if (mainOutputAvailable &&
			    (originalCfg.size == maxYuvSize || yuvCount == 1)) {
				cfg->setStream(const_cast<Stream *>(&data_->outStream_));
				mainOutputAvailable = false;

				pipe.main = cfg->size;
				if (yuvCount == 1)
					pipe.viewfinder = pipe.main;

				LOG(IPU3, Debug) << "Assigned " << cfg->toString()
						 << " to the main output";
			} else {
				cfg->setStream(const_cast<Stream *>(&data_->vfStream_));
				pipe.viewfinder = cfg->size;

				LOG(IPU3, Debug) << "Assigned " << cfg->toString()
						 << " to the viewfinder output";
			}
		}

		if (cfg->pixelFormat != originalCfg.pixelFormat ||
		    cfg->size != originalCfg.size) {
			LOG(IPU3, Debug)
				<< "Stream " << i << " configuration adjusted to "
				<< cfg->toString();
			status = Adjusted;
		}
	}

	/* Only compute the ImgU configuration if a YUV stream has been requested. */
	if (yuvCount) {
		pipeConfig_ = data_->imgu_->calculatePipeConfig(&pipe);
		if (pipeConfig_.isNull()) {
			LOG(IPU3, Error) << "Failed to calculate pipe configuration: "
					 << "unsupported resolutions.";
			return Invalid;
		}
	}

	return status;
}

PipelineHandlerIPU3::PipelineHandlerIPU3(CameraManager *manager)
	: PipelineHandler(manager), cio2MediaDev_(nullptr), imguMediaDev_(nullptr)
{
}

CameraConfiguration *PipelineHandlerIPU3::generateConfiguration(Camera *camera,
								const StreamRoles &roles)
{
	IPU3CameraData *data = cameraData(camera);
	IPU3CameraConfiguration *config = new IPU3CameraConfiguration(data);

	if (roles.empty())
		return config;

	Size sensorResolution = data->cio2_.sensor()->resolution();
	for (const StreamRole role : roles) {
		std::map<PixelFormat, std::vector<SizeRange>> streamFormats;
		unsigned int bufferCount;
		PixelFormat pixelFormat;
		Size size;

		switch (role) {
		case StreamRole::StillCapture:
			/*
			 * Use as default full-frame configuration a value
			 * strictly smaller than the sensor resolution (limited
			 * to the ImgU  maximum output size) and aligned down to
			 * the required frame margin.
			 *
			 * \todo Clarify the alignment constraints as explained
			 * in validate()
			 */
			size = sensorResolution.boundedTo(IMGU_OUTPUT_MAX_SIZE);
			size.width = utils::alignDown(size.width - 1,
						      IMGU_OUTPUT_WIDTH_MARGIN);
			size.height = utils::alignDown(size.height - 1,
						       IMGU_OUTPUT_HEIGHT_MARGIN);
			pixelFormat = formats::NV12;
			bufferCount = IPU3_BUFFER_COUNT;
			streamFormats[pixelFormat] = { { IMGU_OUTPUT_MIN_SIZE, size } };

			break;

		case StreamRole::Raw: {
			StreamConfiguration cio2Config =
				data->cio2_.generateConfiguration(sensorResolution);
			pixelFormat = cio2Config.pixelFormat;
			size = cio2Config.size;
			bufferCount = cio2Config.bufferCount;

			for (const PixelFormat &format : data->cio2_.formats())
				streamFormats[format] = data->cio2_.sizes();

			break;
		}

		case StreamRole::Viewfinder:
		case StreamRole::VideoRecording: {
			/*
			 * Default viewfinder and videorecording to 1280x720,
			 * capped to the maximum sensor resolution and aligned
			 * to the ImgU output constraints.
			 */
			size = sensorResolution.boundedTo(IPU3ViewfinderSize)
					       .alignedDownTo(IMGU_OUTPUT_WIDTH_ALIGN,
							      IMGU_OUTPUT_HEIGHT_ALIGN);
			pixelFormat = formats::NV12;
			bufferCount = IPU3_BUFFER_COUNT;
			streamFormats[pixelFormat] = { { IMGU_OUTPUT_MIN_SIZE, size } };

			break;
		}

		default:
			LOG(IPU3, Error)
				<< "Requested stream role not supported: " << role;
			delete config;
			return nullptr;
		}

		StreamFormats formats(streamFormats);
		StreamConfiguration cfg(formats);
		cfg.size = size;
		cfg.pixelFormat = pixelFormat;
		cfg.bufferCount = bufferCount;
		config->addConfiguration(cfg);
	}

	if (config->validate() == CameraConfiguration::Invalid)
		return {};

	return config;
}

int PipelineHandlerIPU3::configure(Camera *camera, CameraConfiguration *c)
{
	IPU3CameraConfiguration *config =
		static_cast<IPU3CameraConfiguration *>(c);
	IPU3CameraData *data = cameraData(camera);
	Stream *outStream = &data->outStream_;
	Stream *vfStream = &data->vfStream_;
	CIO2Device *cio2 = &data->cio2_;
	ImgUDevice *imgu = data->imgu_;
	V4L2DeviceFormat outputFormat;
	int ret;

	/*
	 * FIXME: enabled links in one ImgU pipe interfere with capture
	 * operations on the other one. This can be easily triggered by
	 * capturing from one camera and then trying to capture from the other
	 * one right after, without disabling media links on the first used
	 * pipe.
	 *
	 * The tricky part here is where to disable links on the ImgU instance
	 * which is currently not in use:
	 * 1) Link enable/disable cannot be done at start()/stop() time as video
	 * devices needs to be linked first before format can be configured on
	 * them.
	 * 2) As link enable has to be done at the least in configure(),
	 * before configuring formats, the only place where to disable links
	 * would be 'stop()', but the Camera class state machine allows
	 * start()<->stop() sequences without any configure() in between.
	 *
	 * As of now, disable all links in the ImgU media graph before
	 * configuring the device, to allow alternate the usage of the two
	 * ImgU pipes.
	 *
	 * As a consequence, a Camera using an ImgU shall be configured before
	 * any start()/stop() sequence. An application that wants to
	 * pre-configure all the camera and then start/stop them alternatively
	 * without going through any re-configuration (a sequence that is
	 * allowed by the Camera state machine) would now fail on the IPU3.
	 */
	ret = imguMediaDev_->disableLinks();
	if (ret)
		return ret;

	/*
	 * \todo: Enable links selectively based on the requested streams.
	 * As of now, enable all links unconditionally.
	 * \todo Don't configure the ImgU at all if we only have a single
	 * stream which is for raw capture, in which case no buffers will
	 * ever be queued to the ImgU.
	 */
	ret = data->imgu_->enableLinks(true);
	if (ret)
		return ret;

	/*
	 * Pass the requested stream size to the CIO2 unit and get back the
	 * adjusted format to be propagated to the ImgU output devices.
	 */
	const Size &sensorSize = config->cio2Format().size;
	V4L2DeviceFormat cio2Format;
	ret = cio2->configure(sensorSize, &cio2Format);
	if (ret)
		return ret;

	CameraSensorInfo sensorInfo;
	cio2->sensor()->sensorInfo(&sensorInfo);
	data->cropRegion_ = sensorInfo.analogCrop;

	/*
	 * If the ImgU gets configured, its driver seems to expect that
	 * buffers will be queued to its outputs, as otherwise the next
	 * capture session that uses the ImgU fails when queueing
	 * buffers to its input.
	 *
	 * If no ImgU configuration has been computed, it means only a RAW
	 * stream has been requested: return here to skip the ImgU configuration
	 * part.
	 */
	ImgUDevice::PipeConfig imguConfig = config->imguConfig();
	if (imguConfig.isNull())
		return 0;

	ret = imgu->configure(imguConfig, &cio2Format);
	if (ret)
		return ret;

	/* Apply the format to the configured streams output devices. */
	StreamConfiguration *mainCfg = nullptr;
	StreamConfiguration *vfCfg = nullptr;

	for (unsigned int i = 0; i < config->size(); ++i) {
		StreamConfiguration &cfg = (*config)[i];
		Stream *stream = cfg.stream();

		if (stream == outStream) {
			mainCfg = &cfg;
			ret = imgu->configureOutput(cfg, &outputFormat);
			if (ret)
				return ret;
		} else if (stream == vfStream) {
			vfCfg = &cfg;
			ret = imgu->configureViewfinder(cfg, &outputFormat);
			if (ret)
				return ret;
		}
	}

	/*
	 * As we need to set format also on the non-active streams, use
	 * the configuration of the active one for that purpose (there should
	 * be at least one active stream in the configuration request).
	 */
	if (!vfCfg) {
		ret = imgu->configureViewfinder(*mainCfg, &outputFormat);
		if (ret)
			return ret;
	}

	/* Apply the "pipe_mode" control to the ImgU subdevice. */
	ControlList ctrls(imgu->imgu_->controls());
	ctrls.set(V4L2_CID_IPU3_PIPE_MODE,
		  static_cast<int32_t>(vfCfg ? IPU3PipeModeVideo :
				       IPU3PipeModeStillCapture));
	ret = imgu->imgu_->setControls(&ctrls);
	if (ret) {
		LOG(IPU3, Error) << "Unable to set pipe_mode control";
		return ret;
	}

	return 0;
}

int PipelineHandlerIPU3::exportFrameBuffers(Camera *camera, Stream *stream,
					    std::vector<std::unique_ptr<FrameBuffer>> *buffers)
{
	IPU3CameraData *data = cameraData(camera);
	unsigned int count = stream->configuration().bufferCount;

	if (stream == &data->outStream_)
		return data->imgu_->output_->exportBuffers(count, buffers);
	else if (stream == &data->vfStream_)
		return data->imgu_->viewfinder_->exportBuffers(count, buffers);
	else if (stream == &data->rawStream_)
		return data->cio2_.exportBuffers(count, buffers);

	return -EINVAL;
}

/**
 * \todo Clarify if 'viewfinder' and 'stat' nodes have to be set up and
 * started even if not in use. As of now, if not properly configured and
 * enabled, the ImgU processing pipeline stalls.
 *
 * In order to be able to start the 'viewfinder' and 'stat' nodes, we need
 * memory to be reserved.
 */
int PipelineHandlerIPU3::allocateBuffers(Camera *camera)
{
	IPU3CameraData *data = cameraData(camera);
	ImgUDevice *imgu = data->imgu_;
	unsigned int bufferCount;
	int ret;

	bufferCount = std::max({
		data->outStream_.configuration().bufferCount,
		data->vfStream_.configuration().bufferCount,
		data->rawStream_.configuration().bufferCount,
	});

	ret = imgu->allocateBuffers(bufferCount);
	if (ret < 0)
		return ret;

	/* Map buffers to the IPA. */
	unsigned int ipaBufferId = 1;

	for (const std::unique_ptr<FrameBuffer> &buffer : imgu->paramBuffers_) {
		buffer->setCookie(ipaBufferId++);
		ipaBuffers_.push_back({
			.id = buffer->cookie(),
			.planes = buffer->planes()
		});
	}

	for (const std::unique_ptr<FrameBuffer> &buffer : imgu->statBuffers_) {
		buffer->setCookie(ipaBufferId++);
		ipaBuffers_.push_back({
			.id = buffer->cookie(),
			.planes = buffer->planes()
		});
	}

	data->ipa_->mapBuffers(ipaBuffers_);

	data->frameInfos_.init(imgu->paramBuffers_, imgu->statBuffers_);

	return 0;
}

int PipelineHandlerIPU3::freeBuffers(Camera *camera)
{
	IPU3CameraData *data = cameraData(camera);

	data->frameInfos_.clear();

	std::vector<unsigned int> ids;
	for (IPABuffer &ipabuf : ipaBuffers_)
		ids.push_back(ipabuf.id);

	data->ipa_->unmapBuffers(ids);
	ipaBuffers_.clear();

	data->imgu_->freeBuffers();

	return 0;
}

int PipelineHandlerIPU3::start(Camera *camera, [[maybe_unused]] ControlList *controls)
{
	IPU3CameraData *data = cameraData(camera);
	CIO2Device *cio2 = &data->cio2_;
	ImgUDevice *imgu = data->imgu_;

	CameraSensorInfo sensorInfo = {};
	std::map<unsigned int, IPAStream> streamConfig;
	std::map<unsigned int, const ControlInfoMap &> entityControls;
	IPAOperationData ipaConfig;
	IPAOperationData result = {};

	int ret;

	/* Allocate buffers for internal pipeline usage. */
	ret = allocateBuffers(camera);
	if (ret)
		return ret;

	IPAOperationData ipaData = {};
	ret = data->ipa_->start(ipaData, nullptr);
	if (ret)
		goto error;

	/*
	 * Start the ImgU video devices, buffers will be queued to the
	 * ImgU output and viewfinder when requests will be queued.
	 */
	ret = cio2->start();
	if (ret)
		goto error;

	ret = imgu->start();
	if (ret)
		goto error;

	/* Inform IPA of stream configuration and sensor controls. */
	ret = data->cio2_.sensor()->sensorInfo(&sensorInfo);
	if (ret)
		goto error;

	streamConfig[0] = {
		.pixelFormat = data->outStream_.configuration().pixelFormat,
		.size = data->outStream_.configuration().size,
	};
	streamConfig[1] = {
		.pixelFormat = data->vfStream_.configuration().pixelFormat,
		.size = data->vfStream_.configuration().size,
	};

	entityControls.emplace(0, data->cio2_.sensor()->controls());

	data->ipa_->configure(sensorInfo, streamConfig, entityControls,
			      ipaConfig, &result);

	return 0;

error:
	imgu->stop();
	cio2->stop();
	data->ipa_->stop();
	freeBuffers(camera);
	LOG(IPU3, Error) << "Failed to start camera " << camera->id();

	return ret;
}

void PipelineHandlerIPU3::stop(Camera *camera)
{
	IPU3CameraData *data = cameraData(camera);
	int ret = 0;

	ret |= data->imgu_->stop();
	ret |= data->cio2_.stop();
	if (ret)
		LOG(IPU3, Warning) << "Failed to stop camera " << camera->id();

	data->ipa_->stop();

	freeBuffers(camera);
}

int PipelineHandlerIPU3::queueRequestDevice(Camera *camera, Request *request)
{
	IPU3CameraData *data = cameraData(camera);

	IPU3Frames::Info *info = data->frameInfos_.create(request);
	if (!info)
		return -ENOBUFS;

	/*
	 * Queue a buffer on the CIO2, using the raw stream buffer provided in
	 * the request, if any, or a CIO2 internal buffer otherwise.
	 */
	FrameBuffer *reqRawBuffer = request->findBuffer(&data->rawStream_);
	FrameBuffer *rawBuffer = data->cio2_.queueBuffer(request, reqRawBuffer);
	if (!rawBuffer) {
		data->frameInfos_.remove(info);
		return -ENOMEM;
	}

	info->rawBuffer = rawBuffer;

	IPAOperationData op;
	op.operation = IPU3_IPA_EVENT_PROCESS_CONTROLS;
	op.data = { info->id };
	op.controls = { request->controls() };
	data->ipa_->processEvent(op);

	return 0;
}

bool PipelineHandlerIPU3::match(DeviceEnumerator *enumerator)
{
	int ret;

	DeviceMatch cio2_dm("ipu3-cio2");
	cio2_dm.add("ipu3-csi2 0");
	cio2_dm.add("ipu3-cio2 0");
	cio2_dm.add("ipu3-csi2 1");
	cio2_dm.add("ipu3-cio2 1");
	cio2_dm.add("ipu3-csi2 2");
	cio2_dm.add("ipu3-cio2 2");
	cio2_dm.add("ipu3-csi2 3");
	cio2_dm.add("ipu3-cio2 3");

	DeviceMatch imgu_dm("ipu3-imgu");
	imgu_dm.add("ipu3-imgu 0");
	imgu_dm.add("ipu3-imgu 0 input");
	imgu_dm.add("ipu3-imgu 0 parameters");
	imgu_dm.add("ipu3-imgu 0 output");
	imgu_dm.add("ipu3-imgu 0 viewfinder");
	imgu_dm.add("ipu3-imgu 0 3a stat");
	imgu_dm.add("ipu3-imgu 1");
	imgu_dm.add("ipu3-imgu 1 input");
	imgu_dm.add("ipu3-imgu 1 parameters");
	imgu_dm.add("ipu3-imgu 1 output");
	imgu_dm.add("ipu3-imgu 1 viewfinder");
	imgu_dm.add("ipu3-imgu 1 3a stat");

	cio2MediaDev_ = acquireMediaDevice(enumerator, cio2_dm);
	if (!cio2MediaDev_)
		return false;

	imguMediaDev_ = acquireMediaDevice(enumerator, imgu_dm);
	if (!imguMediaDev_)
		return false;

	/*
	 * Disable all links that are enabled by default on CIO2, as camera
	 * creation enables all valid links it finds.
	 */
	if (cio2MediaDev_->disableLinks())
		return false;

	ret = imguMediaDev_->disableLinks();
	if (ret)
		return ret;

	ret = registerCameras();

	return ret == 0;
}

/**
 * \brief Initialize the camera controls
 * \param[in] data The camera data
 *
 * Initialize the camera controls as the union of the static pipeline handler
 * controls (IPU3Controls) and controls created dynamically from the sensor
 * capabilities.
 *
 * \return 0 on success or a negative error code otherwise
 */
int PipelineHandlerIPU3::initControls(IPU3CameraData *data)
{
	CameraSensor *sensor = data->cio2_.sensor();
	CameraSensorInfo sensorInfo{};

	int ret = sensor->sensorInfo(&sensorInfo);
	if (ret)
		return ret;

	ControlInfoMap::Map controls = IPU3Controls;

	/*
	 * Compute exposure time limits.
	 *
	 * \todo The exposure limits depend on the sensor configuration.
	 * Initialize the control using the line length and pixel rate of the
	 * current configuration converted to microseconds. Use the
	 * V4L2_CID_EXPOSURE control to get exposure min, max and default and
	 * convert it from lines to microseconds.
	 */
	double lineDuration = sensorInfo.lineLength
			    / (sensorInfo.pixelRate / 1e6);
	const ControlInfoMap &sensorControls = sensor->controls();
	const ControlInfo &v4l2Exposure = sensorControls.find(V4L2_CID_EXPOSURE)->second;
	int32_t minExposure = v4l2Exposure.min().get<int32_t>() * lineDuration;
	int32_t maxExposure = v4l2Exposure.max().get<int32_t>() * lineDuration;
	int32_t defExposure = v4l2Exposure.def().get<int32_t>() * lineDuration;

	/*
	 * \todo Report the actual exposure time, use the default for the
	 * moment.
	 */
	data->exposureTime_ = defExposure;

	controls[&controls::ExposureTime] = ControlInfo(minExposure, maxExposure,
							defExposure);

	/*
	 * Compute the scaler crop limits.
	 *
	 * \todo The scaler crop limits depend on the sensor configuration. It
	 * should be updated when a new configuration is applied. To initialize
	 * the control use the 'Viewfinder' configuration (1280x720) as the
	 * pipeline output resolution and the full sensor size as input frame
	 * (see the todo note in the validate() function about the usage of the
	 * sensor's full frame as ImgU input).
	 */

	/* Re-fetch the sensor info updated to use the largest resolution. */
	V4L2SubdeviceFormat sensorFormat;
	sensorFormat.size = sensor->resolution();
	ret = sensor->setFormat(&sensorFormat);
	if (ret)
		return ret;

	ret = sensor->sensorInfo(&sensorInfo);
	if (ret)
		return ret;

	/*
	 * The maximum scaler crop rectangle is the analogue crop used to
	 * produce the maximum frame size.
	 */
	const Rectangle &analogueCrop = sensorInfo.analogCrop;
	Rectangle maxCrop = analogueCrop;

	/*
	 * As the ImgU cannot up-scale, the minimum selection rectangle has to
	 * be as large as the pipeline output size. Use the default viewfinder
	 * configuration as the desired output size and calculate the minimum
	 * rectangle required to satisfy the ImgU processing margins, unless the
	 * sensor resolution is smaller.
	 *
	 * \todo This implementation is based on the same assumptions about the
	 * ImgU pipeline configuration described in then viewfinder and main
	 * output sizes calculation in the validate() function.
	 */

	/* The strictly smaller size than the sensor resolution, aligned to margins. */
	Size minSize = Size(sensor->resolution().width - 1,
			    sensor->resolution().height - 1)
		       .alignedDownTo(IMGU_OUTPUT_WIDTH_MARGIN,
				      IMGU_OUTPUT_HEIGHT_MARGIN);

	/*
	 * Either the smallest margin-aligned size larger than the viewfinder
	 * size or the adjusted sensor resolution.
	 */
	minSize = Size(IPU3ViewfinderSize.width + 1,
		       IPU3ViewfinderSize.height + 1)
		  .alignedUpTo(IMGU_OUTPUT_WIDTH_MARGIN,
			       IMGU_OUTPUT_HEIGHT_MARGIN)
		  .boundedTo(minSize);

	/*
	 * Re-scale in the sensor's native coordinates. Report (0,0) as
	 * top-left corner as we allow application to freely pan the crop area.
	 */
	Rectangle minCrop = Rectangle(minSize).scaledBy(analogueCrop.size(),
					       sensorInfo.outputSize);

	controls[&controls::ScalerCrop] = ControlInfo(minCrop, maxCrop, maxCrop);

	data->controlInfo_ = std::move(controls);

	return 0;
}

/**
 * \brief Initialise ImgU and CIO2 devices associated with cameras
 *
 * Initialise the two ImgU instances and create cameras with an associated
 * CIO2 device instance.
 *
 * \return 0 on success or a negative error code for error or if no camera
 * has been created
 * \retval -ENODEV no camera has been created
 */
int PipelineHandlerIPU3::registerCameras()
{
	int ret;

	ret = imgu0_.init(imguMediaDev_, 0);
	if (ret)
		return ret;

	ret = imgu1_.init(imguMediaDev_, 1);
	if (ret)
		return ret;

	/*
	 * For each CSI-2 receiver on the IPU3, create a Camera if an
	 * image sensor is connected to it and the sensor can produce images
	 * in a compatible format.
	 */
	unsigned int numCameras = 0;
	for (unsigned int id = 0; id < 4 && numCameras < 2; ++id) {
		std::unique_ptr<IPU3CameraData> data =
			std::make_unique<IPU3CameraData>(this);
		std::set<Stream *> streams = {
			&data->outStream_,
			&data->vfStream_,
			&data->rawStream_,
		};
		CIO2Device *cio2 = &data->cio2_;

		ret = cio2->init(cio2MediaDev_, id);
		if (ret)
			continue;

		ret = data->loadIPA();
		if (ret)
			continue;

		/* Initialize the camera properties. */
		data->properties_ = cio2->sensor()->properties();

		ret = initControls(data.get());
		if (ret)
			continue;

		/*
		 * \todo Read delay values from the sensor itself or from a
		 * a sensor database. For now use generic values taken from
		 * the Raspberry Pi and listed as 'generic values'.
		 */
		std::unordered_map<uint32_t, unsigned int> delays = {
			{ V4L2_CID_ANALOGUE_GAIN, 1 },
			{ V4L2_CID_EXPOSURE, 2 },
		};

		data->delayedCtrls_ =
			std::make_unique<DelayedControls>(cio2->sensor()->device(),
							  delays);
		data->cio2_.frameStart().connect(data->delayedCtrls_.get(),
						 &DelayedControls::applyControls);

		/**
		 * \todo Dynamically assign ImgU and output devices to each
		 * stream and camera; as of now, limit support to two cameras
		 * only, and assign imgu0 to the first one and imgu1 to the
		 * second.
		 */
		data->imgu_ = numCameras ? &imgu1_ : &imgu0_;

		/*
		 * Connect video devices' 'bufferReady' signals to their
		 * slot to implement the image processing pipeline.
		 *
		 * Frames produced by the CIO2 unit are passed to the
		 * associated ImgU input where they get processed and
		 * returned through the ImgU main and secondary outputs.
		 */
		data->cio2_.bufferReady().connect(data.get(),
					&IPU3CameraData::cio2BufferReady);
		data->imgu_->input_->bufferReady.connect(&data->cio2_,
					&CIO2Device::tryReturnBuffer);
		data->imgu_->output_->bufferReady.connect(data.get(),
					&IPU3CameraData::imguOutputBufferReady);
		data->imgu_->viewfinder_->bufferReady.connect(data.get(),
					&IPU3CameraData::imguOutputBufferReady);
Age	Commit message (Collapse)	Author
2020-10-20	android: Omit extra semicolons	Hirokazu Honda
	The end semicolons with LOG_DECLARE_CATEGORY and LOG_DEFINE_CATEGORY are unnecessary. Signed-off-by: Hirokazu Honda <hiroh@chromium.org> Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Niklas Söderlund <niklas.soderlund@ragnatech.se> Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com> Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
2020-10-07	android: camera_device: Return Camera as shared_ptr	Jacopo Mondi
	Return the Camera wrapped by the CameraDevice as a shared_ptr. This will be required to construct the FrameBuffer allocator in the CameraStream class. Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com> Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Signed-off-by: Jacopo Mondi <jacopo@jmondi.org>
2020-09-16	android: Protect against null callbacks	Laurent Pinchart
	According to the Android camera HAL C interface documentation, the camera service is supposed to set callbacks after initializing the HAL and calling get_number_of_cameras(), before any other calls to the module. We rely on this behaviour and use callbacks unconditionally, which would lead to a crash if the camera service behaved incorrectly. While the camera service isn't supposed to behave incorrectly, gracefully handling the error when opening cameras isn't costly, and provides better diagnostic than a crash. While at it, removed an unneeded [[maybe_unused]] attribute. Reported-by: Coverity CID=298638 Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Niklas Söderlund <niklas.soderlund@ragnatech.se> Reviewed-by: Umang Jain <email@uajain.com> Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
2020-08-23	android: camera_hal_manager: Support camera hotplug	Umang Jain
	Extend the support for camera hotplug from libcamera's CameraManager to CameraHalManager. Use camera module callbacks to let the framework know about the hotplug events and change the status of cameras being hotplugged or unplugged via camera_device_status_change(). Introduce a map cameraIdsMap_ which book-keeps all cameras seen in the past by the CameraHalManager. If the camera is seen for the first time, a new id is assigned to it. If the camera has been seen before by the manager, its old id is reused. IDs for internal cameras start with '0' and for external cameras, they start with '1000'. Accesses to cameraIdsMap_ and cameras_ are protected by a mutex. Signed-off-by: Umang Jain <email@uajain.com> Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
2020-08-23	android: camera_device: Make CameraDevice a shared object	Umang Jain
	CameraDevice needs to be wrapper into the std::shared_ptr instead of std::unique_ptr to enable refcounting. The refcounting will help us to support hotplug and hot-unplug CameraHalManager operations in the subsequent commit. Signed-off-by: Umang Jain <email@uajain.com> Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Niklas Söderlund <niklas.soderlund@ragnatech.se> Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
2020-08-23	android: camera_hal_manager: Set camera module callbacks	Umang Jain
	camera_module_callbacks are invoked to inform the framework about the events occurring module-wide. Allow to set these callbacks in camera_hal_manager as this will be used to integration camera hotplug support via camera_module_callbacks::camera_device_status_change in subsequent commit. Signed-off-by: Umang Jain <email@uajain.com> Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Niklas Söderlund <niklas.soderlund@ragnatech.se> Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com> Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
2020-06-04	android: hal_manager: Do not hardcode properties	Jacopo Mondi
	The CameraHalManager::getCameraInfo() method hardcodes the camera facing side and orientation (which corresponds, confusingly, to libcamera's location and rotation properties). Instead of hard-coding the values based on the camera id, inspect the libcamera properties that report the camera location and rotation in a new initialize() method, and use them to report the android camera info and to populate the static metadata buffer. Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Signed-off-by: Jacopo Mondi <jacopo@jmondi.org>
2020-06-04	android: hal_manager: Report supported API version	Jacopo Mondi
	Report the supported API version in the camera_info structure provided to the framework. Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Signed-off-by: Jacopo Mondi <jacopo@jmondi.org>
2020-05-16	libcamera: Move internal headers to include/libcamera/internal/	Laurent Pinchart
	The libcamera internal headers are located in src/libcamera/include/. The directory is added to the compiler headers search path with a meson include_directories() directive, and internal headers are included with (e.g. for the internal semaphore.h header) #include "semaphore.h" All was well, until libcxx decided to implement the C++20 synchronization library. The __threading_support header gained a #include <semaphore.h> to include the pthread's semaphore support. As include_directories() adds src/libcamera/include/ to the compiler search path with -I, the internal semaphore.h is included instead of the pthread version. Needless to say, the compiler isn't happy. Three options have been considered to fix this issue: - Use -iquote instead of -I. The -iquote option instructs gcc to only consider the header search path for headers included with the "" version. Meson unfortunately doesn't support this option. - Rename the internal semaphore.h header. This was deemed to be the beginning of a long whack-a-mole game, where namespace clashes with system libraries would appear over time (possibly dependent on particular system configurations) and would need to be constantly fixed. - Move the internal headers to another directory to create a unique namespace through path components. This causes lots of churn in all the existing source files through the all project. The first option would be best, but isn't available to us due to missing support in meson. Even if -iquote support was added, we would need to fix the problem before a new version of meson containing the required support would be released. The third option is thus the only practical solution available. Bite the bullet, and do it, moving headers to include/libcamera/internal/. Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Acked-by: Jacopo Mondi <jacopo@jmondi.org>
2020-02-13	android: Remove internal thread	Laurent Pinchart
	Now that libcamera creates threads internally and doesn't rely on an application-provided event loop, remove the thread from the Android Camera HAL layer. The CameraProxy class becomes meaningless, remove it and communicate directly from the CameraHalManager to the CameraDevice. Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Acked-by: Jacopo Mondi <jacopo@jmondi.org>
2019-08-19	libcamera: camera_manager: Construct CameraManager instances manually	Laurent Pinchart
	The CameraManager class is not supposed to be instantiated multiple times, which led to a singleton implementation. This requires a global instance of the CameraManager, which is destroyed when the global destructors are executed. Relying on global instances causes issues with cleanup, as the order in which the global destructors are run can't be controlled. In particular, the Android camera HAL implementation ends up destroying the CameraHalManager after the CameraManager, which leads to use-after-free problems. To solve this, remove the CameraManager::instance() method and make the CameraManager class instantiable directly. Multiple instances are still not allowed, and this is enforced by storing the instance pointer internally to be checked when an instance is created. Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Jacopo Mondi <jacopo@jmondi.org>
2019-08-19	android: camera_hal_manager: Clean up resources when terminating	Laurent Pinchart
	The CameraHalManager starts the libcamera CameraManager and creates CameraProxy instances for each camera in the system. Clean up those resources when the CameraHalManager terminates. Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Jacopo Mondi <jacopo@jmondi.org>
2019-08-19	android: camera_hal_manager: Remove unused close() method	Laurent Pinchart
	The CameraHalManager::close() method isn't used, remove it. Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Jacopo Mondi <jacopo@jmondi.org>
2019-08-19	android: camera_hal_manager: Stop thread when destroying	Laurent Pinchart
	The CameraHalManager starts a thread that is never stopped. This leads to the thread being destroyed while running, which causes a crash. Fix this by stopping the thread and waiting for it to finish in the destructor of the CameraHalManager. Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Jacopo Mondi <jacopo@jmondi.org>
2019-08-15	hal: Fix comparison of unsigned integer < 0	Laurent Pinchart
	The CameraHalManager::getCameraInfo() validates the camera id it receives from the camera service, and in doing so generates a compiler error with gcc as the id is an unsigned integer and can never be negative: ../src/android/camera_hal_manager.cpp: In member function ‘CameraProxy* CameraHalManager::open(unsigned int, const hw_module_t*)’: ../src/android/camera_hal_manager.cpp:89:9: error: comparison of unsigned expression < 0 is always false [-Werror=type-limits] if (id < 0 \|\| id >= numCameras()) { Fix it by removing the unneeded comparison. Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Jacopo Mondi <jacopo@jmondi.org>
2019-08-12	hal: Fix comparison of integers of different signs	Laurent Pinchart
	The CameraHalManager::getCameraInfo() validates the camera id it receives from the camera service, and in doing so compares it with an unsigned integer, generating a compiler error: src/android/camera_hal_manager.cpp:121:9: error: comparison of integers of different signs: 'int' and 'unsigned int' [-Werror,-Wsign-compare] if (id >= numCameras() \|\| id < 0) { ~~ ^ ~~~~~~~~~~~~ Fix this by turning the id into an unsigned int, as camera ids can't be negative. If a negative id is received from the camera service it will be converted to a large unsigned integer that will fail the comparison with numCameras(). Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> Reviewed-by: Niklas Söderlund <niklas.soderlund@ragnatech.se> Reviewed-by: Jacopo Mondi <jacopo@jmondi.org>
2019-08-12	android: hal: Add Camera3 HAL	Jacopo Mondi
	Add libcamera Android Camera HALv3 implementation. The initial camera HAL implementation supports the LIMITED hardware level and uses statically defined metadata and camera characteristics. Add a build option named 'android' and adjust the build system to selectively compile the Android camera HAL and link it against the required Android libraries. Signed-off-by: Jacopo Mondi <jacopo@jmondi.org>