libcamera/pinchartl/libcamera.git - Laurent Pinchart's clone of libcamera

diff options

-rw-r--r--

2 files changed, 16 insertions, 33 deletions

/* SPDX-License-Identifier: LGPL-2.1-or-later */ /* * Copyright (C) 2019-2020, Raspberry Pi (Trading) Ltd. * * raspberrypi.cpp - Pipeline handler for Raspberry Pi devices */ #include <algorithm> #include <assert.h> #include <fcntl.h> #include <mutex> #include <queue> #include <sys/mman.h> #include <libcamera/camera.h> #include <libcamera/control_ids.h> #include <libcamera/file_descriptor.h> #include <libcamera/formats.h> #include <libcamera/ipa/raspberrypi.h> #include <libcamera/logging.h> #include <libcamera/property_ids.h> #include <libcamera/request.h> #include <linux/videodev2.h> #include "libcamera/internal/camera_sensor.h" #include "libcamera/internal/device_enumerator.h" #include "libcamera/internal/ipa_manager.h" #include "libcamera/internal/media_device.h" #include "libcamera/internal/pipeline_handler.h" #include "libcamera/internal/utils.h" #include "libcamera/internal/v4l2_controls.h" #include "libcamera/internal/v4l2_videodevice.h" #include "dma_heaps.h" #include "rpi_stream.h" #include "staggered_ctrl.h" namespace libcamera { LOG_DEFINE_CATEGORY(RPI) namespace { bool isRaw(PixelFormat &pixFmt) { /* * The isRaw test might be redundant right now the pipeline handler only * supports RAW sensors. Leave it in for now, just as a sanity check. */ const PixelFormatInfo &info = PixelFormatInfo::info(pixFmt); if (!info.isValid()) return false; return info.colourEncoding == PixelFormatInfo::ColourEncodingRAW; } double scoreFormat(double desired, double actual) { double score = desired - actual; /* Smaller desired dimensions are preferred. */ if (score < 0.0) score = (-score) / 8; /* Penalise non-exact matches. */ if (actual != desired) score *= 2; return score; } V4L2DeviceFormat findBestMode(V4L2VideoDevice::Formats &formatsMap, const Size &req) { double bestScore = std::numeric_limits<double>::max(), score; V4L2DeviceFormat bestMode = {}; #define PENALTY_AR 1500.0 #define PENALTY_8BIT 2000.0 #define PENALTY_10BIT 1000.0 #define PENALTY_12BIT 0.0 #define PENALTY_UNPACKED 500.0 /* Calculate the closest/best mode from the user requested size. */ for (const auto &iter : formatsMap) { V4L2PixelFormat v4l2Format = iter.first; const PixelFormatInfo &info = PixelFormatInfo::info(v4l2Format); for (const SizeRange &sz : iter.second) { double modeWidth = sz.contains(req) ? req.width : sz.max.width; double modeHeight = sz.contains(req) ? req.height : sz.max.height; double reqAr = static_cast<double>(req.width) / req.height; double modeAr = modeWidth / modeHeight; /* Score the dimensions for closeness. */ score = scoreFormat(req.width, modeWidth); score += scoreFormat(req.height, modeHeight); score += PENALTY_AR * scoreFormat(reqAr, modeAr); /* Add any penalties... this is not an exact science! */ if (!info.packed) score += PENALTY_UNPACKED; if (info.bitsPerPixel == 12) score += PENALTY_12BIT; else if (info.bitsPerPixel == 10) score += PENALTY_10BIT; else if (info.bitsPerPixel == 8) score += PENALTY_8BIT; if (score <= bestScore) { bestScore = score; bestMode.fourcc = v4l2Format; bestMode.size = Size(modeWidth, modeHeight); } LOG(RPI, Info) << "Mode: " << modeWidth << "x" << modeHeight << " fmt " << v4l2Format.toString() << " Score: " << score << " (best " << bestScore << ")"; } } return bestMode; } enum class Unicam : unsigned int { Image, Embedded }; enum class Isp : unsigned int { Input, Output0, Output1, Stats }; } /* namespace */ class RPiCameraData : public CameraData { public: RPiCameraData(PipelineHandler *pipe) : CameraData(pipe), sensor_(nullptr), state_(State::Stopped), dropFrameCount_(0), ispOutputCount_(0) { } void frameStarted(uint32_t sequence); int loadIPA(); int configureIPA(); void queueFrameAction(unsigned int frame, const IPAOperationData &action); /* bufferComplete signal handlers. */ void unicamBufferDequeue(FrameBuffer *buffer); void ispInputDequeue(FrameBuffer *buffer); void ispOutputDequeue(FrameBuffer *buffer); void clearIncompleteRequests(); void handleStreamBuffer(FrameBuffer *buffer, RPi::RPiStream *stream); void handleState(); CameraSensor *sensor_; /* Array of Unicam and ISP device streams and associated buffers/streams. */ RPi::RPiDevice<Unicam, 2> unicam_; RPi::RPiDevice<Isp, 4> isp_; /* The vector below is just for convenience when iterating over all streams. */ std::vector<RPi::RPiStream *> streams_; /* Buffers passed to the IPA. */ std::vector<IPABuffer> ipaBuffers_; /* DMAHEAP allocation helper. */ RPi::DmaHeap dmaHeap_; FileDescriptor lsTable_; RPi::StaggeredCtrl staggeredCtrl_; uint32_t expectedSequence_; bool sensorMetadata_; /* * All the functions in this class are called from a single calling * thread. So, we do not need to have any mutex to protect access to any * of the variables below. */ enum class State { Stopped, Idle, Busy, IpaComplete }; State state_; std::queue<FrameBuffer *> bayerQueue_; std::queue<FrameBuffer *> embeddedQueue_; std::deque<Request *> requestQueue_; unsigned int dropFrameCount_; private: void checkRequestCompleted(); void tryRunPipeline(); void tryFlushQueues(); FrameBuffer *updateQueue(std::queue<FrameBuffer *> &q, uint64_t timestamp, RPi::RPiStream *stream); unsigned int ispOutputCount_; }; class RPiCameraConfiguration : public CameraConfiguration { public: RPiCameraConfiguration(const RPiCameraData *data); Status validate() override; private: const RPiCameraData *data_; }; class PipelineHandlerRPi : public PipelineHandler { public: PipelineHandlerRPi(CameraManager *manager); CameraConfiguration *generateConfiguration(Camera *camera, const StreamRoles &roles) override; int configure(Camera *camera, CameraConfiguration *config) override; int exportFrameBuffers(Camera *camera, Stream *stream, std::vector<std::unique_ptr<FrameBuffer>> *buffers) override; int start(Camera *camera) override; void stop(Camera *camera) override; int queueRequestDevice(Camera *camera, Request *request) override; bool match(DeviceEnumerator *enumerator) override; private: RPiCameraData *cameraData(const Camera *camera) { return static_cast<RPiCameraData *>(PipelineHandler::cameraData(camera)); } int queueAllBuffers(Camera *camera); int prepareBuffers(Camera *camera); void freeBuffers(Camera *camera); MediaDevice *unicam_; MediaDevice *isp_; }; RPiCameraConfiguration::RPiCameraConfiguration(const RPiCameraData *data) : CameraConfiguration(), data_(data) { } CameraConfiguration::Status RPiCameraConfiguration::validate() { Status status = Valid; if (config_.empty()) return Invalid; unsigned int rawCount = 0, outCount = 0, count = 0, maxIndex = 0; std::pair<int, Size> outSize[2]; Size maxSize; for (StreamConfiguration &cfg : config_) { if (isRaw(cfg.pixelFormat)) { /* * Calculate the best sensor mode we can use based on * the user request. */ V4L2VideoDevice::Formats fmts = data_->unicam_[Unicam::Image].dev()->formats(); V4L2DeviceFormat sensorFormat = findBestMode(fmts, cfg.size); int ret = data_->unicam_[Unicam::Image].dev()->tryFormat(&sensorFormat); if (ret) return Invalid; PixelFormat sensorPixFormat = sensorFormat.fourcc.toPixelFormat(); if (cfg.size != sensorFormat.size || cfg.pixelFormat != sensorPixFormat) { cfg.size = sensorFormat.size; cfg.pixelFormat = sensorPixFormat; status = Adjusted; } cfg.stride = sensorFormat.planes[0].bpl; cfg.frameSize = sensorFormat.planes[0].size; rawCount++; } else { outSize[outCount] = std::make_pair(count, cfg.size); /* Record the largest resolution for fixups later. */ if (maxSize < cfg.size) { maxSize = cfg.size; maxIndex = outCount; } outCount++; } count++; /* Can only output 1 RAW stream, or 2 YUV/RGB streams. */ if (rawCount > 1 || outCount > 2) { LOG(RPI, Error) << "Invalid number of streams requested"; return Invalid; } } /* * Now do any fixups needed. For the two ISP outputs, one stream must be * equal or smaller than the other in all dimensions. */ for (unsigned int i = 0; i < outCount; i++) { outSize[i].second.width = std::min(outSize[i].second.width, maxSize.width); outSize[i].second.height = std::min(outSize[i].second.height, maxSize.height); if (config_.at(outSize[i].first).size != outSize[i].second) { config_.at(outSize[i].first).size = outSize[i].second; status = Adjusted; } /* * Also validate the correct pixel formats here. * Note that Output0 and Output1 support a different * set of formats. * * Output 0 must be for the largest resolution. We will * have that fixed up in the code above. * */ StreamConfiguration &cfg = config_.at(outSize[i].first); PixelFormat &cfgPixFmt = cfg.pixelFormat; V4L2VideoDevice *dev; if (i == maxIndex) dev = data_->isp_[Isp::Output0].dev(); else dev = data_->isp_[Isp::Output1].dev(); V4L2VideoDevice::Formats fmts = dev->formats(); if (fmts.find(V4L2PixelFormat::fromPixelFormat(cfgPixFmt, false)) == fmts.end()) { /* If we cannot find a native format, use a default one. */ cfgPixFmt = formats::NV12; status = Adjusted; } V4L2DeviceFormat format = {}; format.fourcc = dev->toV4L2PixelFormat(cfg.pixelFormat); format.size = cfg.size; int ret = dev->tryFormat(&format); if (ret) return Invalid; cfg.stride = format.planes[0].bpl; cfg.frameSize = format.planes[0].size; } return status; } PipelineHandlerRPi::PipelineHandlerRPi(CameraManager *manager) : PipelineHandler(manager), unicam_(nullptr), isp_(nullptr) { } CameraConfiguration *PipelineHandlerRPi::generateConfiguration(Camera *camera, const StreamRoles &roles) { RPiCameraData *data = cameraData(camera); CameraConfiguration *config = new RPiCameraConfiguration(data); V4L2DeviceFormat sensorFormat; unsigned int bufferCount; PixelFormat pixelFormat; V4L2VideoDevice::Formats fmts; Size size; if (roles.empty()) return config; unsigned int rawCount = 0; unsigned int outCount = 0; for (const StreamRole role : roles) { switch (role) { case StreamRole::StillCaptureRaw: size = data->sensor_->resolution(); fmts = data->unicam_[Unicam::Image].dev()->formats(); sensorFormat = findBestMode(fmts, size); pixelFormat = sensorFormat.fourcc.toPixelFormat(); ASSERT(pixelFormat.isValid()); bufferCount = 2; rawCount++; break; case StreamRole::StillCapture: fmts = data->isp_[Isp::Output0].dev()->formats(); pixelFormat = formats::NV12; /* Return the largest sensor resolution. */ size = data->sensor_->resolution(); bufferCount = 1; outCount++; break; case StreamRole::VideoRecording: fmts = data->isp_[Isp::Output0].dev()->formats(); pixelFormat = formats::NV12; size = { 1920, 1080 }; bufferCount = 4; outCount++; break; case StreamRole::Viewfinder: fmts = data->isp_[Isp::Output0].dev()->formats(); pixelFormat = formats::ARGB8888; size = { 800, 600 }; bufferCount = 4; outCount++; break; default: LOG(RPI, Error) << "Requested stream role not supported: " << role; delete config; return nullptr; } if (rawCount > 1 || outCount > 2) { LOG(RPI, Error) << "Invalid stream roles requested"; delete config; return nullptr; } /* Translate the V4L2PixelFormat to PixelFormat. */ std::map<PixelFormat, std::vector<SizeRange>> deviceFormats; for (const auto &format : fmts) { PixelFormat pixelFormat = format.first.toPixelFormat(); if (pixelFormat.isValid()) deviceFormats[pixelFormat] = format.second; } /* Add the stream format based on the device node used for the use case. */ StreamFormats formats(deviceFormats); StreamConfiguration cfg(formats); cfg.size = size; cfg.pixelFormat = pixelFormat; cfg.bufferCount = bufferCount; config->addConfiguration(cfg); } config->validate(); return config; } int PipelineHandlerRPi::configure(Camera *camera, CameraConfiguration *config) { RPiCameraData *data = cameraData(camera); int ret; /* Start by resetting the Unicam and ISP stream states. */ for (auto const stream : data->streams_) stream->reset(); Size maxSize, sensorSize; unsigned int maxIndex = 0; bool rawStream = false; /* * Look for the RAW stream (if given) size as well as the largest * ISP output size. */ for (unsigned i = 0; i < config->size(); i++) { StreamConfiguration &cfg = config->at(i); if (isRaw(cfg.pixelFormat)) { /* * If we have been given a RAW stream, use that size * for setting up the sensor. */ sensorSize = cfg.size; rawStream = true; } else { if (cfg.size > maxSize) { maxSize = config->at(i).size; maxIndex = i; } } } /* First calculate the best sensor mode we can use based on the user request. */ V4L2VideoDevice::Formats fmts = data->unicam_[Unicam::Image].dev()->formats(); V4L2DeviceFormat sensorFormat = findBestMode(fmts, rawStream ? sensorSize : maxSize); /* * Unicam image output format. The ISP input format gets set at start, * just in case we have swapped bayer orders due to flips. */ ret = data->unicam_[Unicam::Image].dev()->setFormat(&sensorFormat); if (ret) return ret; LOG(RPI, Info) << "Sensor: " << camera->id() << " - Selected mode: " << sensorFormat.toString(); /* * This format may be reset on start() if the bayer order has changed * because of flips in the sensor. */ ret = data->isp_[Isp::Input].dev()->setFormat(&sensorFormat); /* * See which streams are requested, and route the user * StreamConfiguration appropriately. */ V4L2DeviceFormat format = {}; for (unsigned i = 0; i < config->size(); i++) { StreamConfiguration &cfg = config->at(i); if (isRaw(cfg.pixelFormat)) { cfg.setStream(&data->unicam_[Unicam::Image]); /* * We must set both Unicam streams as external, even * though the application may only request RAW frames. * This is because we match timestamps on both streams * to synchronise buffers. */ data->unicam_[Unicam::Image].setExternal(true); data->unicam_[Unicam::Embedded].setExternal(true); continue; } if (i == maxIndex) { /* ISP main output format. */ V4L2VideoDevice *dev = data->isp_[Isp::Output0].dev(); V4L2PixelFormat fourcc = dev->toV4L2PixelFormat(cfg.pixelFormat); format.size = cfg.size; format.fourcc = fourcc; ret = dev->setFormat(&format); if (ret) return -EINVAL; if (format.size != cfg.size || format.fourcc != fourcc) { LOG(RPI, Error) << "Failed to set format on ISP capture0 device: " << format.toString(); return -EINVAL; } cfg.setStream(&data->isp_[Isp::Output0]); data->isp_[Isp::Output0].setExternal(true); } /* * ISP second output format. This fallthrough means that if a * second output stream has not been configured, we simply use * the Output0 configuration. */ V4L2VideoDevice *dev = data->isp_[Isp::Output1].dev(); format.fourcc = dev->toV4L2PixelFormat(cfg.pixelFormat); format.size = cfg.size; ret = dev->setFormat(&format); if (ret) { LOG(RPI, Error) << "Failed to set format on ISP capture1 device: " << format.toString(); return ret; } /* * If we have not yet provided a stream for this config, it * means this is to be routed from Output1. */ if (!cfg.stream()) { cfg.setStream(&data->isp_[Isp::Output1]); data->isp_[Isp::Output1].setExternal(true); } } /* ISP statistics output format. */ format = {}; format.fourcc = V4L2PixelFormat(V4L2_META_FMT_BCM2835_ISP_STATS); ret = data->isp_[Isp::Stats].dev()->setFormat(&format); if (ret) { LOG(RPI, Error) << "Failed to set format on ISP stats stream: " << format.toString(); return ret; } /* Unicam embedded data output format. */ format = {}; format.fourcc = V4L2PixelFormat(V4L2_META_FMT_SENSOR_DATA); LOG(RPI, Debug) << "Setting embedded data format."; ret = data->unicam_[Unicam::Embedded].dev()->setFormat(&format); if (ret) { LOG(RPI, Error) << "Failed to set format on Unicam embedded: " << format.toString(); return ret; } /* Adjust aspect ratio by providing crops on the input image. */ Rectangle crop{ 0, 0, sensorFormat.size }; int ar = maxSize.height * sensorFormat.size.width - maxSize.width * sensorFormat.size.height; if (ar > 0) crop.width = maxSize.width * sensorFormat.size.height / maxSize.height; else if (ar < 0) crop.height = maxSize.height * sensorFormat.size.width / maxSize.width; crop.width &= ~1; crop.height &= ~1; crop.x = (sensorFormat.size.width - crop.width) >> 1; crop.y = (sensorFormat.size.height - crop.height) >> 1; data->isp_[Isp::Input].dev()->setSelection(V4L2_SEL_TGT_CROP, &crop); ret = data->configureIPA(); if (ret) LOG(RPI, Error) << "Failed to configure the IPA: " << ret; return ret; } int PipelineHandlerRPi::exportFrameBuffers([[maybe_unused]] Camera *camera, Stream *stream, std::vector<std::unique_ptr<FrameBuffer>> *buffers) { RPi::RPiStream *s = static_cast<RPi::RPiStream *>(stream); unsigned int count = stream->configuration().bufferCount; int ret = s->dev()->exportBuffers(count, buffers); s->setExportedBuffers(buffers); return ret; } int PipelineHandlerRPi::start(Camera *camera) { RPiCameraData *data = cameraData(camera); int ret; /* Allocate buffers for internal pipeline usage. */ ret = prepareBuffers(camera); if (ret) { LOG(RPI, Error) << "Failed to allocate buffers"; stop(camera); return ret; } ret = queueAllBuffers(camera); if (ret) { LOG(RPI, Error) << "Failed to queue buffers"; stop(camera); return ret; } /* Start the IPA. */ ret = data->ipa_->start(); if (ret) { LOG(RPI, Error) << "Failed to start IPA for " << camera->id(); stop(camera); return ret; } /* * IPA configure may have changed the sensor flips - hence the bayer * order. Get the sensor format and set the ISP input now. */ V4L2DeviceFormat sensorFormat; data->unicam_[Unicam::Image].dev()->getFormat(&sensorFormat); ret = data->isp_[Isp::Input].dev()->setFormat(&sensorFormat); if (ret) { stop(camera); return ret; } /* Enable SOF event generation. */ data->unicam_[Unicam::Image].dev()->setFrameStartEnabled(true); /* * Write the last set of gain and exposure values to the camera before * starting. First check that the staggered ctrl has been initialised * by configure(). */ ASSERT(data->staggeredCtrl_); data->staggeredCtrl_.reset(); data->staggeredCtrl_.write(); data->expectedSequence_ = 0; data->state_ = RPiCameraData::State::Idle; /* Start all streams. */ for (auto const stream : data->streams_) { ret = stream->dev()->streamOn(); if (ret) { stop(camera); return ret; } } return 0; } void PipelineHandlerRPi::stop(Camera *camera) { RPiCameraData *data = cameraData(camera); data->state_ = RPiCameraData::State::Stopped; /* Disable SOF event generation. */ data->unicam_[Unicam::Image].dev()->setFrameStartEnabled(false); /* This also stops the streams. */ data->clearIncompleteRequests(); data->bayerQueue_ = {}; data->embeddedQueue_ = {}; /* Stop the IPA. */ data->ipa_->stop(); freeBuffers(camera); } int PipelineHandlerRPi::queueRequestDevice(Camera *camera, Request *request) { RPiCameraData *data = cameraData(camera); if (data->state_ == RPiCameraData::State::Stopped) return -EINVAL; LOG(RPI, Debug) << "queueRequestDevice: New request."; /* Push all buffers supplied in the Request to the respective streams. */ for (auto stream : data->streams_) { if (stream->isExternal()) { FrameBuffer *buffer = request->findBuffer(stream); /* * If no buffer is provided by the request for this stream, we * queue a nullptr to the stream to signify that it must use an * internally allocated buffer for this capture request. This * buffer will not be given back to the application, but is used * to support the internal pipeline flow. * * The below queueBuffer() call will do nothing if there are not * enough internal buffers allocated, but this will be handled by * queuing the request for buffers in the RPiStream object. */ int ret = stream->queueBuffer(buffer); if (ret)


context:
space:
mode: