libcamera: imx8-isi: Split Bayer/YUV config generationisi/raw_sensor

The ISI can:
- produce any processed (YUV/RGB) stream format from a non-Raw media bus
  format
- produce Bayer Raw stream formats from a Bayer RAW media bus format

At generateConfiguration() a YUV/RGB pixel format is preferred for the
StillCapture/VideoRecording/Viewfinder roles, but currently there are no
guarantees in place that the sensor provide a non-Bayer bus format from
which YUV/RGB an be generated.

This makes the default configuration generated for those roles not to
work if the sensor is a RAW-only one.

To improve the situation split the configuration generation in two:
- One for YUV modes
- One for Raw Bayer mode

StreamRoles assigned to a YUV mode will try to first generate a YUV
configuration and then fallback to RAW if that's what the sensor can
provide.

As an additional requirement, for YUV streams, the generated mode has to
be validated with the sensor to confirm the desired sizes can be
generated. In order to test a format on the sensor introduce
CameraSensor::tryFormat().

Reported-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Signed-off-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>

3 files changed, 188 insertions, 76 deletions

diff --git a/include/libcamera/internal/camera_sensor.h b/include/libcamera/internal/camera_sensor.h
index b9f4d786..ce3a790f 100644
--- a/include/libcamera/internal/camera_sensor.h
+++ b/include/libcamera/internal/camera_sensor.h
@@ -54,6 +54,7 @@ public:
 	V4L2SubdeviceFormat getFormat(const std::vector<unsigned int> &mbusCodes,
 				      const Size &size) const;
 	int setFormat(V4L2SubdeviceFormat *format);
+	int tryFormat(V4L2SubdeviceFormat *format) const;
 
 	const ControlInfoMap &controls() const;
 	ControlList getControls(const std::vector<uint32_t> &ids);
diff --git a/src/libcamera/camera_sensor.cpp b/src/libcamera/camera_sensor.cpp
index a210aa4f..dfe59303 100644
--- a/src/libcamera/camera_sensor.cpp
+++ b/src/libcamera/camera_sensor.cpp
@@ -758,6 +758,20 @@ int CameraSensor::setFormat(V4L2SubdeviceFormat *format)
 }
 
 /**
+ * \brief Try the sensor output format
+ * \param[in] format The desired sensor output format
+ *
+ * The ranges of any controls associated with the sensor are not updated.
+ *
+ * \return 0 on success or a negative error code otherwise
+ */
+int CameraSensor::tryFormat(V4L2SubdeviceFormat *format) const
+{
+	return subdev_->setFormat(pad_, format,
+				  V4L2Subdevice::Whence::TryFormat);
+}
+
+/**
  * \brief Retrieve the supported V4L2 controls and their information
  *
  * Control information is updated automatically to reflect the current sensor
diff --git a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp
index 0c67e35d..883e5a08 100644
--- a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp
+++ b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp
@@ -141,6 +141,10 @@ private:
 
 	Pipe *pipeFromStream(Camera *camera, const Stream *stream);
 
+	StreamConfiguration generateYUVConfiguration(Camera *camera,
+						     const Size &size);
+	StreamConfiguration generateRawConfiguration(Camera *camera);
+
 	void bufferReady(FrameBuffer *buffer);
 
 	MediaDevice *isiDev_;
@@ -569,6 +573,147 @@ PipelineHandlerISI::PipelineHandlerISI(CameraManager *manager)
 {
 }
 
+/*
+ * Generate a StreamConfiguration for YUV/RGB use case.
+ *
+ * Pick the first YUV/RGB format the sensor can produce as default and, if the
+ * sensor can produce YUV/RGB, collect all the processed pixel formats the
+ * ISI can generate as supported stream configurations.
+ */
+StreamConfiguration PipelineHandlerISI::generateYUVConfiguration(Camera *camera,
+								 const Size &size)
+{
+	const CameraSensor *sensor = cameraData(camera)->sensor_.get();
+	PixelFormat pixelFormat;
+	Size sensorSize;
+
+	for (unsigned int code : sensor->mbusCodes()) {
+
+		/* Find a YUV/RGB media bus code from the sensor. */
+		const BayerFormat &bayerFormat = BayerFormat::fromMbusCode(code);
+		if (bayerFormat.isValid())
+			continue;
+
+		/* Make sure the media format is listed in formatsMap_. */
+		auto it = std::find_if(ISICameraConfiguration::formatsMap_.begin(),
+				       ISICameraConfiguration::formatsMap_.end(),
+				       [code](auto &isiFormat) {
+						auto &pipe = isiFormat.second;
+						return pipe.sensorCode == code;
+				       });
+
+		if (it == ISICameraConfiguration::formatsMap_.end()) {
+			LOG(ISI, Warning) << utils::hex(code)
+					  << " not supported in formats map.";
+			continue;
+		}
+
+		/* Adjust the requested size to the sensor's capabilities. */
+		V4L2SubdeviceFormat sensorFmt;
+		sensorFmt.mbus_code = code;
+		sensorFmt.size = size;
+
+		int ret = sensor->tryFormat(&sensorFmt);
+		if (ret) {
+			LOG(ISI, Error) << "Failed to try sensor format.";
+			return {};
+		}
+
+		sensorSize = sensorFmt.size;
+
+		break;
+	}
+
+	if (!pixelFormat.isValid())
+		return {};
+
+	/*
+	 * Populate the StreamConfiguration.
+	 *
+	 * As the sensor supports at least one YUV/RGB format all the processed
+	 * ones in formatsMap_ can be generated from it.
+	 */
+	std::map<PixelFormat, std::vector<SizeRange>> streamFormats;
+
+	for (const auto &[pixFmt, pipeFmt] : ISICameraConfiguration::formatsMap_) {
+		const PixelFormatInfo &info = PixelFormatInfo::info(pixFmt);
+		if (info.colourEncoding == PixelFormatInfo::ColourEncodingRAW)
+			continue;
+
+		streamFormats[pixFmt] = { { kMinISISize, sensorSize } };
+	}
+
+	StreamFormats formats(streamFormats);
+
+	StreamConfiguration cfg(formats);
+	cfg.pixelFormat = pixelFormat;
+	cfg.size = sensorSize;
+	cfg.bufferCount = 4;
+
+	return cfg;
+}
+
+/*
+ * Generate a StreamConfiguration for Raw Bayer use case.
+ *
+ * The Raw Bayer format with the largest bit depth is selected as default and
+ * one stream configuration is collected for each Raw Bayer format the sensor
+ * can produce.
+ */
+StreamConfiguration PipelineHandlerISI::generateRawConfiguration(Camera *camera)
+{
+	const CameraSensor *sensor = cameraData(camera)->sensor_.get();
+	std::map<PixelFormat, std::vector<SizeRange>> streamFormats;
+	unsigned int maxDepth = 0;
+	PixelFormat pixelFormat;
+
+	for (unsigned int code : sensor->mbusCodes()) {
+
+		/* Find a Bayer media bus code from the sensor. */
+		const BayerFormat &bayerFormat = BayerFormat::fromMbusCode(code);
+		if (!bayerFormat.isValid())
+			continue;
+
+		/* Make sure the media format is listed in formatsMap_. */
+		auto it = std::find_if(ISICameraConfiguration::formatsMap_.begin(),
+				       ISICameraConfiguration::formatsMap_.end(),
+				       [code](auto &isiFormat) {
+						auto &pipe = isiFormat.second;
+						return pipe.sensorCode == code;
+				       });
+
+		if (it == ISICameraConfiguration::formatsMap_.end()) {
+			LOG(ISI, Warning) << bayerFormat
+					  << " not supported in formats map.";
+			continue;
+		}
+
+		/*
+		 * Populate the StreamConfiguration with all the supported Bayer
+		 * formats the sensor can produce.
+		 */
+		streamFormats[it->first] = { { kMinISISize, sensor->resolution() } };
+
+		/* And pick the one with the largest bit depth as default. */
+		if (bayerFormat.bitDepth > maxDepth) {
+			maxDepth = bayerFormat.bitDepth;
+			pixelFormat = it->first;
+		}
+	}
+
+	if (!pixelFormat.isValid())
+		return {};
+
+	StreamFormats formats(streamFormats);
+
+	StreamConfiguration cfg(formats);
+	cfg.size = sensor->resolution();
+	cfg.pixelFormat = pixelFormat;
+	cfg.bufferCount = 4;
+
+	return cfg;
+}
+
 std::unique_ptr<CameraConfiguration>
 PipelineHandlerISI::generateConfiguration(Camera *camera,
 					  const StreamRoles &roles)
@@ -586,79 +731,45 @@ PipelineHandlerISI::generateConfiguration(Camera *camera,
 		return nullptr;
 	}
 
-	bool isRaw = false;
 	for (const auto &role : roles) {
 		/*
-		 * Prefer the following formats
+		 * Prefer the following formats:
 		 * - Still Capture: Full resolution YUYV
 		 * - ViewFinder/VideoRecording: 1080p YUYV
-		 * - RAW: sensor's native format and resolution
+		 * - RAW: Full resolution Bayer
 		 */
-		std::map<PixelFormat, std::vector<SizeRange>> streamFormats;
-		PixelFormat pixelFormat;
-		Size size;
+		StreamConfiguration cfg;
 
 		switch (role) {
 		case StreamRole::StillCapture:
-			/*
-			 * \todo Make sure the sensor can produce non-RAW formats
-			 * compatible with the ones supported by the pipeline.
-			 */
-			size = data->sensor_->resolution();
-			pixelFormat = formats::YUYV;
+			cfg = generateYUVConfiguration(camera,
+						       data->sensor_->resolution());
+			if (!cfg.pixelFormat.isValid()) {
+				/*
+				 * Fallback to use a Bayer format if that's what
+				 * the sensor supports.
+				 */
+				cfg = generateRawConfiguration(camera);
+			}
+
 			break;
 
 		case StreamRole::Viewfinder:
 		case StreamRole::VideoRecording:
-			/*
-			 * \todo Make sure the sensor can produce non-RAW formats
-			 * compatible with the ones supported by the pipeline.
-			 */
-			size = PipelineHandlerISI::kPreviewSize;
-			pixelFormat = formats::YUYV;
-			break;
-
-		case StreamRole::Raw: {
-			/*
-			 * Make sure the sensor can generate a RAW format and
-			 * prefer the ones with a larger bitdepth.
-			 */
-			const ISICameraConfiguration::FormatMap::value_type *rawPipeFormat = nullptr;
-			unsigned int maxDepth = 0;
-
-			for (unsigned int code : data->sensor_->mbusCodes()) {
-				const BayerFormat &bayerFormat = BayerFormat::fromMbusCode(code);
-				if (!bayerFormat.isValid())
-					continue;
-
-				/* Make sure the format is supported by the pipeline handler. */
-				auto it = std::find_if(ISICameraConfiguration::formatsMap_.begin(),
-						       ISICameraConfiguration::formatsMap_.end(),
-						       [code](auto &isiFormat) {
-							        auto &pipe = isiFormat.second;
-							        return pipe.sensorCode == code;
-						       });
-				if (it == ISICameraConfiguration::formatsMap_.end())
-					continue;
-
-				if (bayerFormat.bitDepth > maxDepth) {
-					maxDepth = bayerFormat.bitDepth;
-					rawPipeFormat = &(*it);
-				}
-			}
-
-			if (!rawPipeFormat) {
-				LOG(ISI, Error)
-					<< "Cannot generate a configuration for RAW stream";
-				return nullptr;
+			cfg = generateYUVConfiguration(camera,
+						       PipelineHandlerISI::kPreviewSize);
+			if (!cfg.pixelFormat.isValid()) {
+				/*
+				 * Fallback to use a Bayer format if that's what
+				 * the sensor supports.
+				 */
+				cfg = generateRawConfiguration(camera);
 			}
 
-			size = data->sensor_->resolution();
-			pixelFormat = rawPipeFormat->first;
-
-			streamFormats[pixelFormat] = { { kMinISISize, size } };
-			isRaw = true;
+			break;
 
+		case StreamRole::Raw: {
+			cfg = generateRawConfiguration(camera);
 			break;
 		}
 
@@ -667,26 +778,12 @@ PipelineHandlerISI::generateConfiguration(Camera *camera,
 			return nullptr;
 		}
 
-		/*
-		 * For non-RAW configurations the ISI can perform colorspace
-		 * conversion. List all the supported output formats here.
-		 */
-		if (!isRaw) {
-			for (const auto &[pixFmt, pipeFmt] : ISICameraConfiguration::formatsMap_) {
-				const PixelFormatInfo &info = PixelFormatInfo::info(pixFmt);
-				if (info.colourEncoding == PixelFormatInfo::ColourEncodingRAW)
-					continue;
-
-				streamFormats[pixFmt] = { { kMinISISize, size } };
-			}
+		if (!cfg.pixelFormat.isValid()) {
+			LOG(ISI, Error)
+				<< "Cannot generate configuration for role: " << role;
+			return nullptr;
 		}
 
-		StreamFormats formats(streamFormats);
-
-		StreamConfiguration cfg(formats);
-		cfg.pixelFormat = pixelFormat;
-		cfg.size = size;
-		cfg.bufferCount = 4;
 		config->addConfiguration(cfg);
 	}