/* SPDX-License-Identifier: LGPL-2.1-or-later */ /* * Copyright (C) 2018, Google Inc. * * camera.cpp - Camera device */ #include #include #include #include #include "log.h" #include "pipeline_handler.h" #include "utils.h" /** * \file camera.h * \brief Camera device handling * * At the core of libcamera is the camera device, combining one image source * with processing hardware able to provide one or multiple image streams. The * Camera class represents a camera device. * * A camera device contains a single image source, and separate camera device * instances relate to different image sources. For instance, a phone containing * front and back image sensors will be modelled with two camera devices, one * for each sensor. When multiple streams can be produced from the same image * source, all those streams are guaranteed to be part of the same camera * device. * * While not sharing image sources, separate camera devices can share other * system resources, such as an ISP. For this reason camera device instances may * not be fully independent, in which case usage restrictions may apply. For * instance, a phone with a front and a back camera device may not allow usage * of the two devices simultaneously. */ namespace libcamera { LOG_DECLARE_CATEGORY(Camera) /** * \class CameraConfiguration * \brief Hold configuration for streams of the camera * The CameraConfiguration holds an ordered list of stream configurations. It * supports iterators and operates as a vector of StreamConfiguration instances. * The stream configurations are inserted by addConfiguration(), and the * operator[](int) returns a reference to the StreamConfiguration based on its * insertion index. Accessing a stream configuration with an invalid index * results in undefined behaviour. * * CameraConfiguration instances are retrieved from the camera with * Camera::generateConfiguration(). Applications may then inspect the * configuration, modify it, and possibly add new stream configuration entries * with addConfiguration(). Once the camera configuration satisfies the * application, it shall be validated by a call to validate(). The validation * implements "try" semantics: it adjusts invalid configurations to the closest * achievable parameters instead of rejecting them completely. Applications * then decide whether to accept the modified configuration, or try again with * a different set of parameters. Once the configuration is valid, it is passed * to Camera::configure(). */ /** * \enum CameraConfiguration::Status * \brief Validity of a camera configuration * \var CameraConfiguration::Valid * The configuration is fully valid * \var CameraConfiguration::Adjusted * The configuration has been adjusted to a valid configuration * \var CameraConfiguration::Invalid * The configuration is invalid and can't be adjusted automatically */ /** * \typedef CameraConfiguration::iterator * \brief Iterator for the stream configurations in the camera configuration */ /** * \typedef CameraConfiguration::const_iterator * \brief Const iterator for the stream configuration in the camera * configuration */ /** * \brief Create an empty camera configuration */ CameraConfiguration::CameraConfiguration() : config_({}) { } CameraConfiguration::~CameraConfiguration() { } /** * \brief Add a stream configuration to the camera configuration * \param[in] cfg The stream configuration */ void CameraConfiguration::addConfiguration(const StreamConfiguration &cfg) { config_.push_back(cfg); } /** * \fn CameraConfiguration::validate() * \brief Validate and possibly adjust the camera configuration * * This method adjusts the camera configuration to the closest valid * configuration and returns the validation status. * * \todo: Define exactly when to return each status code. Should stream * parameters set to 0 by the caller be adjusted without returning Adjusted ? * This would potentially be useful for applications but would get in the way * in Camera::configure(). Do we need an extra status code to signal this ? * * \todo: Handle validation of buffers count when refactoring the buffers API. * * \return A CameraConfiguration::Status value that describes the validation * status. * \retval CameraConfiguration::Invalid The configuration is invalid and can't * be adjusted. This may only occur in extreme cases such as when the * configuration is empty. * \retval CameraConfigutation::Adjusted The configuration has been adjusted * and is now valid. Parameters may have changed for any stream, and stream * configurations may have been removed. The caller shall check the * configuration carefully. * \retval CameraConfiguration::Valid The configuration was already valid and * hasn't been adjusted. */ /** * \brief Retrieve a reference to a stream configuration * \param[in] index Numerical index * * The \a index represents the zero based insertion order of stream * configuration into the camera configuration with addConfiguration(). Calling * this method with an invalid index results in undefined behaviour. * * \return The stream configuration */ StreamConfiguration &CameraConfiguration::at(unsigned int index) { return config_[index]; } /** * \brief Retrieve a const reference to a stream configuration * \param[in] index Numerical index * * The \a index represents the zero based insertion order of stream * configuration into the camera configuration with addConfiguration(). Calling * this method with an invalid index results in undefined behaviour. * * \return The stream configuration */ const StreamConfiguration &CameraConfiguration::at(unsigned int index) const { return config_[index]; } /** * \fn StreamConfiguration &CameraConfiguration::operator[](unsigned int) * \brief Retrieve a reference to a stream configuration * \param[in] index Numerical index * * The \a index represents the zero based insertion order of stream * configuration into the camera configuration with addConfiguration(). Calling * this method with an invalid index results in undefined behaviour. * * \return The stream configuration */ /** * \fn const StreamConfiguration &CameraConfiguration::operator[](unsigned int) const * \brief Retrieve a const reference to a stream configuration * \param[in] index Numerical index * * The \a index represents the zero based insertion order of stream * configuration into the camera configuration with addConfiguration(). Calling * this method with an invalid index results in undefined behaviour. * * \return The stream configuration */ /** * \brief Retrieve an iterator to the first stream configuration in the * sequence * \return An iterator to the first stream configuration */ CameraConfiguration::iterator CameraConfiguration::begin() { return config_.begin(); } /** * \brief Retrieve a const iterator to the first element of the stream * configurations * \return A const iterator to the first stream configuration */ CameraConfiguration::const_iterator CameraConfiguration::begin() const { return config_.begin(); } /** * \brief Retrieve an iterator pointing to the past-the-end stream * configuration in the sequence * \return An iterator to the element following the last stream configuration */ CameraConfiguration::iterator CameraConfiguration::end() { return config_.end(); } /** * \brief Retrieve a const iterator pointing to the past-the-end stream * configuration in the sequence * \return A const iterator to the element following the last stream * configuration */ CameraConfiguration::const_iterator CameraConfiguration::end() const { return config_.end(); } /** * \brief Check if the camera configuration is empty * \return True if the configuration is empty */ bool CameraConfiguration::empty() const { return config_.empty(); } /** * \brief Retrieve the number of stream configurations * \return Number of stream configurations */ std::size_t CameraConfiguration::size() const { return config_.size(); } /** * \var CameraConfiguration::config_ * \brief The vector of stream configurations */ /** * \class Camera * \brief Camera device * * \todo Add documentation for camera start timings. What exactly does the * camera expect the pipeline handler to do when start() is called? * * The Camera class models a camera capable of producing one or more image * streams from a single image source. It provides the main interface to * configuring and controlling the device, and capturing image streams. It is * the central object exposed by libcamera. * * To support the central nature of Camera objects, libcamera manages the * lifetime of camera instances with std::shared_ptr<>. Instances shall be * created with the create() function which returns a shared pointer. The * Camera constructors and destructor are private, to prevent instances from * being constructed and destroyed manually. * * \section camera_operation Operating the Camera * * An application needs to perform a sequence of operations on a camera before * it is ready to process requests. The camera needs to be acquired, configured * and resources allocated or imported to prepare the camera for capture. Once * started the camera can process requests until it is stopped. When an * application is done with a camera all resources allocated need to be freed * and the camera released. * * An application may start and stop a camera multiple times as long as it is * not released. The camera may also be reconfigured provided that all * resources allocated are freed prior to the reconfiguration. * * \subsection Camera States * * To help manage the sequence of operations needed to control the camera a set * of states are defined. Each state describes which operations may be performed * on the camera. Operations not listed in the state diagram are allowed in all * states. * * \dot * digraph camera_state_machine { * node [shape = doublecircle ]; Available; * node [shape = circle ]; Acquired; * node [shape = circle ]; Configured; * node [shape = circle ]; Prepared; * node [shape = circle ]; Running; * * Available -> Available [label = "release()"]; * Available -> Acquired [label = "acquire()"]; * * Acquired -> Available [label = "release()"]; * Acquired -> Configured [label = "configure()"]; * * Configured -> Available [label = "release()"]; * Configured -> Configured [label = "configure()"]; * Configured -> Prepared [label = "allocateBuffers()"]; * * Prepared -> Configured [label = "freeBuffers()"]; * Prepared -> Prepared [label = "createRequest()"]; * Prepared -> Running [label = "start()"]; * * Running -> Prepared [label = "stop()"]; * Running -> Running [label = "createRequest(), queueRequest()"]; * } * \enddot * * \subsubsection Available * The base state of a camera, an application can inspect the properties of the * camera to determine if it wishes to use it. If an application wishes to use * a camera it should acquire() it to proceed to the Acquired state. * * \subsubsection Acquired * In the acquired state an application has exclusive access to the camera and * may modify the camera's parameters to configure it and proceed to the * Configured state. * * \subsubsection Configured * The camera is configured and ready for the application to prepare it with * resources. The camera may be reconfigured multiple times until resources * are provided and the state progresses to Prepared. * * \subsubsection Prepared * The camera has been configured and provided with resources and is ready to be * started. The application may free the camera's resources to get back to the * Configured state or start() it to progress to the Running state. * * \subsubsection Running * The camera is running and ready to process requests queued by the * application. The camera remains in this state until it is stopped and moved * to the Prepared state. */ /** * \brief Create a camera instance * \param[in] name The name of the camera device * \param[in] pipe The pipeline handler responsible for the camera device * \param[in] streams Array of streams the camera provides * * The caller is responsible for guaranteeing unicity of the camera name. * * \return A shared pointer to the newly created camera object */ std::shared_ptr Camera::create(PipelineHandler *pipe, const std::string &name, const std::set &streams) { struct Deleter : std::default_delete { void operator()(Camera *camera) { delete camera; } }; Camera *camera = new Camera(pipe, name); camera->streams_ = streams; return std::shared_ptr(camera, Deleter()); } /** * \brief Retrieve the name of the camera * \return Name of the camera device */ const std::string &Camera::name() const { return name_; } /** * \var Camera::bufferCompleted * \brief Signal emitted when a buffer for a request queued to the camera has * completed */ /** * \var Camera::requestCompleted * \brief Signal emitted when a request queued to the camera has completed */ /** * \var Camera::disconnected * \brief Signal emitted when the camera is disconnected from the system * * This signal is emitted when libcamera detects that the camera has been * removed from the system. For hot-pluggable devices this is usually caused by * physical device disconnection. The media device is passed as a parameter. * * As soon as this signal is emitted the camera instance will refuse all new * application API calls by returning errors immediately. */ Camera::Camera(PipelineHandler *pipe, const std::string &name) : pipe_(pipe->shared_from_this()), name_(name), disconnected_(false), state_(CameraAvailable) { } Camera::~Camera() { if (!stateIs(CameraAvailable)) LOG(Camera, Error) << "Removing camera while still in use"; } static const char *const camera_state_names[] = { "Available", "Acquired", "Configured", "Prepared", "Running", }; bool Camera::stateBetween(State low, State high) const { if (state_ >= low && state_ <= high) return true; ASSERT(static_cast(low) < ARRAY_SIZE(camera_state_names) && static_cast(high) < ARRAY_SIZE(camera_state_names)); LOG(Camera, Debug) << "Camera in " << camera_state_names[state_] << " state trying operation requiring state between " << camera_state_names[low] << " and " << camera_state_names[high]; return false; } bool Camera::stateIs(State state) const { if (state_ == state) return true; ASSERT(static_cast(state) < ARRAY_SIZE(camera_state_names)); LOG(Camera, Debug) << "Camera in " << camera_state_names[state_] << " state trying operation requiring state " << camera_state_names[state]; return false; } /** * \brief Notify camera disconnection * * This method is used to notify the camera instance that the underlying * hardware has been unplugged. In response to the disconnection the camera * instance notifies the application by emitting the #disconnected signal, and * ensures that all new calls to the application-facing Camera API return an * error immediately. * * \todo Deal with pending requests if the camera is disconnected in a * running state. * \todo Update comment about Running state when importing buffers as well as * allocating them are supported. */ void Camera::disconnect() { LOG(Camera, Debug) << "Disconnecting camera " << name_; /* * If the camera was running when the hardware was removed force the * state to Prepared to allow applications to call freeBuffers() and * release() before deleting the camera. */ if (state_ == CameraRunning) state_ = CameraPrepared; disconnected_ = true; disconnected.emit(this); } /** * \brief Acquire the camera device for exclusive access * * After opening the device with open(), exclusive access must be obtained * before performing operations that change the device state. This function is * not blocking, if the device has already been acquired (by the same or another * process) the -EBUSY error code is returned. * * Acquiring a camera will limit usage of any other camera(s) provided by the * same pipeline handler to the same instance of libcamera. The limit is in * effect until all cameras from the pipeline handler are released. Other * instances of libcamera can still list and examine the cameras but will fail * if they attempt to acquire() any of them. * * Once exclusive access isn't needed anymore, the device should be released * with a call to the release() function. * * This function affects the state of the camera, see \ref camera_operation. * * \return 0 on success or a negative error code otherwise * \retval -ENODEV The camera has been disconnected from the system * \retval -EBUSY The camera is not free and can't be acquired by the caller */ int Camera::acquire() { if (disconnected_) return -ENODEV; if (!stateIs(CameraAvailable)) return -EBUSY; if (!pipe_->lock()) { LOG(Camera, Info) << "Pipeline handler in use by another process"; return -EBUSY; } state_ = CameraAcquired; return 0; } /** * \brief Release exclusive access to the camera device * * Releasing the camera device allows other users to acquire exclusive access * with the acquire() function. * * This function affects the state of the camera, see \ref camera_operation. * * \return 0 on success or a negative error code otherwise * \retval -EBUSY The camera is running and can't be released */ int Camera::release() { if (!stateBetween(CameraAvailable, CameraConfigured)) return -EBUSY; pipe_->unlock(); state_ = CameraAvailable; return 0; } /** * \brief Retrieve all the camera's stream information * * Retrieve all of the camera's static stream information. The static * information describes among other things how many streams the camera * supports and the capabilities of each stream. * * \return An array of all the camera's streams. */ const std::set &Camera::streams() const { return streams_; } /** * \brief Generate a default camera configuration according to stream roles * \param[in] roles A list of stream roles * * Generate a camera configuration for a set of desired stream roles. The caller * specifies a list of stream roles and the camera returns a configuration * containing suitable streams and their suggested default configurations. An * empty list of roles is valid, and will generate an empty configuration that * can be filled by the caller. * * \return A CameraConfiguration if the requested roles can be satisfied, or a * null pointer otherwise. The ownership of the returned configuration is * passed to the caller. */ std::unique_ptr Camera::generateConfiguration(const StreamRoles &roles) { if (disconnected_ || roles.size() > streams_.size()) return nullptr; CameraConfiguration *config = pipe_->generateConfiguration(this, roles); if (!config) { LOG(Camera, Debug) << "Pipeline handler failed to generate configuration"; return nullptr; } std::ostringstream msg("streams configuration:", std::ios_base::ate); if (config->empty()) msg << " empty"; for (unsigned int index = 0; index < config->size(); ++index) msg << " (" << index << ") " << config->at(index).toString(); LOG(Camera, Debug) << msg.str(); return std::unique_ptr(config); } /** * \brief Configure the camera prior to capture * \param[in] config The camera configurations to setup * * Prior to starting capture, the camera must be configured to select a * group of streams to be involved in the capture and their configuration. * The caller specifies which streams are to be involved and their configuration * by populating \a config. * * The configuration is created by generateConfiguration(), and adjusted by the * caller with CameraConfiguration::validate(). This method only accepts fully * valid configurations and returns an error if \a config is not valid. * * Exclusive access to the camera shall be ensured by a call to acquire() prior * to calling this function, otherwise an -EACCES error will be returned. * * This function affects the state of the camera, see \ref camera_operation. * * Upon return the StreamConfiguration entries in \a config are associated with * Stream instances which can be retrieved with StreamConfiguration::stream(). * * \return 0 on success or a negative error code otherwise * \retval -ENODEV The camera has been disconnected from the system * \retval -EACCES The camera is not in a state where it can be configured * \retval -EINVAL The configuration is not valid */ int Camera::configure(CameraConfiguration *config) { int ret; if (disconnected_) return -ENODEV; if (!stateBetween(CameraAcquired, CameraConfigured)) return -EACCES; if (config->validate() != CameraConfiguration::Valid) { LOG(Camera, Error) << "Can't configure camera with invalid configuration"; return -EINVAL; } std::ostringstream msg("configuring streams:", std::ios_base::ate); for (unsigned int index = 0; index < config->size(); ++index) { StreamConfiguration &cfg = config->at(index); cfg.setStream(nullptr); msg << " (" << index << ") " << cfg.toString(); } LOG(Camera, Info) << msg.str(); ret = pipe_->configure(this, config); if (ret) return ret; activeStreams_.clear(); for (const StreamConfiguration &cfg : *config) { Stream *stream = cfg.stream(); if (!stream) LOG(Camera, Fatal) << "Pipeline handler failed to update stream configuration"; stream->configuration_ = cfg; activeStreams_.insert(stream); /* * Allocate buffer objects in the pool. * Memory will be allocated and assigned later. */ stream->bufferPool().createBuffers(cfg.bufferCount); } state_ = CameraConfigured; return 0; } /** * \brief Allocate buffers for all configured streams * * This function affects the state of the camera, see \ref camera_operation. * * \return 0 on success or a negative error code otherwise * \retval -ENODEV The camera has been disconnected from the system * \retval -EACCES The camera is not in a state where buffers can be allocated * \retval -EINVAL The configuration is not valid */ int Camera::allocateBuffers() { if (disconnected_) return -ENODEV; if (!stateIs(CameraConfigured)) return -EACCES; if (activeStreams_.empty()) { LOG(Camera, Error) << "Can't allocate buffers without streams"; return -EINVAL; } int ret = pipe_->allocateBuffers(this, activeStreams_); if (ret) { LOG(Camera, Error) << "Failed to allocate buffers"; return ret; } state_ = CameraPrepared; return 0; } /** * \brief Release all buffers from allocated pools in each stream * * This function affects the state of the camera, see \ref camera_operation. * * \return 0 on success or a negative error code otherwise * \retval -EACCES The camera is not in a state where buffers can be freed */ int Camera::freeBuffers() { if (!stateIs(CameraPrepared)) return -EACCES; for (Stream *stream : activeStreams_) { if (!stream->bufferPool().count()) continue; /* * All mappings must be destroyed before buffers can be freed * by the V4L2 device that has allocated them. */ stream->bufferPool().destroyBuffers(); } state_ = CameraConfigured; return pipe_->freeBuffers(this, activeStreams_); } /** * \brief Create a request object for the camera * * This method creates an empty request for the application to fill with * buffers and paramaters, and queue for capture. * * The ownership of the returned request is passed to the caller, which is * responsible for either queueing the request or deleting it. * * This function shall only be called when the camera is in the Prepared * or Running state, see \ref camera_operation. * * \return A pointer to the newly created request, or nullptr on error */ Request *Camera::createRequest() { if (disconnected_ || !stateBetween(CameraPrepared, CameraRunning)) return nullptr; return new Request(this); } /** * \brief Queue a request to the camera * \param[in] request The request to queue to the camera * * This method queues a \a request to the camera for capture. * * After allocating the request with createRequest(), the application shall * fill it with at least one capture buffer before queuing it. Requests that * contain no buffers are invalid and are rejected without being queued. * * Once the request has been queued, the camera will notify its completion * through the \ref requestCompleted signal. * * Ownership of the request is transferred to the camera. It will be deleted * automatically after it completes. * * \return 0 on success or a negative error code otherwise * \retval -ENODEV The camera has been disconnected from the system * \retval -EACCES The camera is not running so requests can't be queued * \retval -EINVAL The request is invalid */ int Camera::queueRequest(Request *request) { if (disconnected_) return -ENODEV; if (!stateIs(CameraRunning)) return -EACCES; for (auto const &it : request->buffers()) { Stream *stream = it.first; if (activeStreams_.find(stream) == activeStreams_.end()) { LOG(Camera, Error) << "Invalid request"; return -EINVAL; } } int ret = request->prepare(); if (ret) { LOG(Camera, Error) << "Failed to prepare request"; return ret; } return pipe_->queueRequest(this, request); } /** * \brief Start capture from camera * * Start the camera capture session. Once the camera is started the application * can queue requests to the camera to process and return to the application * until the capture session is terminated with \a stop(). * * This function affects the state of the camera, see \ref camera_operation. * * \return 0 on success or a negative error code otherwise * \retval -ENODEV The camera has been disconnected from the system * \retval -EACCES The camera is not in a state where it can be started */ int Camera::start() { if (disconnected_) return -ENODEV; if (!stateIs(CameraPrepared)) return -EACCES; LOG(Camera, Debug) << "Starting capture"; int ret = pipe_->start(this); if (ret) return ret; state_ = CameraRunning; return 0; } /** * \brief Stop capture from camera * * This method stops capturing and processing requests immediately. All pending * requests are cancelled and complete synchronously in an error state. * * This function affects the state of the camera, see \ref camera_operation. * * \return 0 on success or a negative error code otherwise * \retval -ENODEV The camera has been disconnected from the system * \retval -EACCES The camera is not running so can't be stopped */ int Camera::stop() { if (disconnected_) return -ENODEV; if (!stateIs(CameraRunning)) return -EACCES; LOG(Camera, Debug) << "Stopping capture"; state_ = CameraPrepared; pipe_->stop(this); return 0; } /** * \brief Handle request completion and notify application * \param[in] request The request that has completed * * This function is called by the pipeline handler to notify the camera that * the request has completed. It emits the requestCompleted signal and deletes * the request. */ void Camera::requestComplete(Request *request) { std::map buffers(std::move(request->bufferMap_)); requestCompleted.emit(request, buffers); delete request; } } /* namespace libcamera */ l num">1 ''' print metadata for debug ''' def print_meta(self): print('\nData:') print(' ver = {}'.format(self.ver)) print(' w = {}'.format(self.w)) print(' h = {}'.format(self.h)) print(' pad = {}'.format(self.pad)) print(' fmt = {}'.format(self.fmt)) print(' sigbits = {}'.format(self.sigbits)) print(' pattern = {}'.format(self.pattern)) print(' exposure = {}'.format(self.exposure)) print(' againQ8 = {}'.format(self.againQ8)) print(' againQ8_norm = {}'.format(self.againQ8_norm)) print(' camName = {}'.format(self.camName)) print(' blacklevel = {}'.format(self.blacklevel)) print(' blacklevel_16 = {}'.format(self.blacklevel_16)) return 1 """ get image from raw scanline data """ def get_image(self, raw): self.dptr = [] """ check if data is 10 or 12 bits """ if self.sigbits == 10: """ calc length of scanline """ lin_len = ((((((self.w+self.pad+3)>>2)) * 5)+31)>>5) * 32 """ stack scan lines into matrix """ raw = np.array(raw).reshape(-1, lin_len).astype(np.int64)[:self.h, ...] """ separate 5 bits in each package, stopping when w is satisfied """ ba0 = raw[..., 0:5*((self.w+3)>>2):5] ba1 = raw[..., 1:5*((self.w+3)>>2):5] ba2 = raw[..., 2:5*((self.w+3)>>2):5] ba3 = raw[..., 3:5*((self.w+3)>>2):5] ba4 = raw[..., 4:5*((self.w+3)>>2):5] """ assemble 10 bit numbers """ ch0 = np.left_shift((np.left_shift(ba0, 2) + (ba4 % 4)), 6) ch1 = np.left_shift((np.left_shift(ba1, 2) + (np.right_shift(ba4, 2) % 4)), 6) ch2 = np.left_shift((np.left_shift(ba2, 2) + (np.right_shift(ba4, 4) % 4)), 6) ch3 = np.left_shift((np.left_shift(ba3, 2) + (np.right_shift(ba4, 6) % 4)), 6) """ interleave bits """ mat = np.empty((self.h, self.w), dtype=ch0.dtype) mat[..., 0::4] = ch0 mat[..., 1::4] = ch1 mat[..., 2::4] = ch2 mat[..., 3::4] = ch3 """ There is som eleaking memory somewhere in the code. This code here seemed to make things good enough that the code would run for reasonable numbers of images, however this is techincally just a workaround. (sorry) """ ba0, ba1, ba2, ba3, ba4 = None, None, None, None, None del ba0, ba1, ba2, ba3, ba4 ch0, ch1, ch2, ch3 = None, None, None, None del ch0, ch1, ch2, ch3 """ same as before but 12 bit case """ elif self.sigbits == 12: lin_len = ((((((self.w+self.pad+1)>>1)) * 3)+31)>>5) * 32 raw = np.array(raw).reshape(-1, lin_len).astype(np.int64)[:self.h, ...] ba0 = raw[..., 0:3*((self.w+1)>>1):3] ba1 = raw[..., 1:3*((self.w+1)>>1):3] ba2 = raw[..., 2:3*((self.w+1)>>1):3] ch0 = np.left_shift((np.left_shift(ba0, 4) + ba2 % 16), 4) ch1 = np.left_shift((np.left_shift(ba1, 4) + (np.right_shift(ba2, 4)) % 16), 4) mat = np.empty((self.h, self.w), dtype=ch0.dtype) mat[..., 0::2] = ch0 mat[..., 1::2] = ch1 else: """ data is neither 10 nor 12 or incorrect data """ print('ERROR: wrong bit format, only 10 or 12 bit supported') return 0 """ separate bayer channels """ c0 = mat[0::2, 0::2] c1 = mat[0::2, 1::2] c2 = mat[1::2, 0::2] c3 = mat[1::2, 1::2] self.channels = [c0, c1, c2, c3] return 1 """ obtain 16x16 patch centred at macbeth square centre for each channel """ def get_patches(self, cen_coords, size=16): """ obtain channel widths and heights """ ch_w, ch_h = self.w, self.h cen_coords = list(np.array((cen_coords[0])).astype(np.int32)) self.cen_coords = cen_coords """ squares are ordered by stacking macbeth chart columns from left to right. Some useful patch indices: white = 3 black = 23 'reds' = 9, 10 'blues' = 2, 5, 8, 20, 22 'greens' = 6, 12, 17 greyscale = 3, 7, 11, 15, 19, 23 """ all_patches = [] for ch in self.channels: ch_patches = [] for cen in cen_coords: ''' macbeth centre is placed at top left of central 2x2 patch to account for rounding Patch pixels are sorted by pixel brightness so spatial information is lost. ''' patch = ch[cen[1]-7:cen[1]+9, cen[0]-7:cen[0]+9].flatten() patch.sort() if patch[-5] == (2**self.sigbits-1)*2**(16-self.sigbits): self.saturated = True ch_patches.append(patch) # print('\nNew Patch\n') all_patches.append(ch_patches) # print('\n\nNew Channel\n\n') self.patches = all_patches return 1 def brcm_load_image(Cam, im_str): """ Load image where raw data and metadata is in the BRCM format """ try: """ create byte array """ with open(im_str, 'rb') as image: f = image.read() b = bytearray(f) """ return error if incorrect image address """ except FileNotFoundError: print('\nERROR:\nInvalid image address') Cam.log += '\nWARNING: Invalid image address' return 0 """ return error if problem reading file """ if f is None: print('\nERROR:\nProblem reading file') Cam.log += '\nWARNING: Problem readin file' return 0 # print('\nLooking for EOI and BRCM header') """ find end of image followed by BRCM header by turning bytearray into hex string and string matching with regexp """ start = -1 match = bytearray(b'\xff\xd9@BRCM') match_str = binascii.hexlify(match) b_str = binascii.hexlify(b) """ note index is divided by two to go from string to hex """ indices = [m.start()//2 for m in re.finditer(match_str, b_str)] # print(indices) try: start = indices[0] + 3 except IndexError: print('\nERROR:\nNo Broadcom header found') Cam.log += '\nWARNING: No Broadcom header found!' return 0 """ extract data after header """ # print('\nExtracting data after header') buf = b[start:start+32768] Img = Image(buf) Img.str = im_str # print('Data found successfully') """ obtain metadata """ # print('\nReading metadata') Img.get_meta() Cam.log += '\nExposure : {} us'.format(Img.exposure) Cam.log += '\nNormalised gain : {}'.format(Img.againQ8_norm) # print('Metadata read successfully') """ obtain raw image data """ # print('\nObtaining raw image data') raw = b[start+32768:] Img.get_image(raw) """ delete raw to stop memory errors """ raw = None del raw # print('Raw image data obtained successfully') return Img def dng_load_image(Cam, im_str): try: Img = Image(None) # RawPy doesn't load all the image tags that we need, so we use py3exiv2 metadata = pyexif.ImageMetadata(im_str) metadata.read() Img.ver = 100 # random value """ The DNG and TIFF/EP specifications use different IFDs to store the raw image data and the Exif tags. DNG stores them in a SubIFD and in an Exif IFD respectively (named "SubImage1" and "Photo" by pyexiv2), while TIFF/EP stores them both in IFD0 (name "Image"). Both are used in "DNG" files, with libcamera-apps following the DNG recommendation and applications based on picamera2 following TIFF/EP. This code detects which tags are being used, and therefore extracts the correct values. """ try: Img.w = metadata['Exif.SubImage1.ImageWidth'].value subimage = "SubImage1" photo = "Photo" except KeyError: Img.w = metadata['Exif.Image.ImageWidth'].value subimage = "Image" photo = "Image" Img.pad = 0 Img.h = metadata[f'Exif.{subimage}.ImageLength'].value white = metadata[f'Exif.{subimage}.WhiteLevel'].value Img.sigbits = int(white).bit_length() Img.fmt = (Img.sigbits - 4) // 2 Img.exposure = int(metadata[f'Exif.{photo}.ExposureTime'].value * 1000000) Img.againQ8 = metadata[f'Exif.{photo}.ISOSpeedRatings'].value * 256 / 100 Img.againQ8_norm = Img.againQ8 / 256 Img.camName = metadata['Exif.Image.Model'].value Img.blacklevel = int(metadata[f'Exif.{subimage}.BlackLevel'].value[0]) Img.blacklevel_16 = Img.blacklevel << (16 - Img.sigbits) bayer_case = { '0 1 1 2': (0, (0, 1, 2, 3)), '1 2 0 1': (1, (2, 0, 3, 1)), '2 1 1 0': (2, (3, 2, 1, 0)), '1 0 2 1': (3, (1, 0, 3, 2)) } cfa_pattern = metadata[f'Exif.{subimage}.CFAPattern'].value Img.pattern = bayer_case[cfa_pattern][0] Img.order = bayer_case[cfa_pattern][1] # Now use RawPy tp get the raw Bayer pixels raw_im = raw.imread(im_str) raw_data = raw_im.raw_image shift = 16 - Img.sigbits c0 = np.left_shift(raw_data[0::2, 0::2].astype(np.int64), shift) c1 = np.left_shift(raw_data[0::2, 1::2].astype(np.int64), shift) c2 = np.left_shift(raw_data[1::2, 0::2].astype(np.int64), shift) c3 = np.left_shift(raw_data[1::2, 1::2].astype(np.int64), shift) Img.channels = [c0, c1, c2, c3] Img.rgb = raw_im.postprocess() except Exception: print("\nERROR: failed to load DNG file", im_str) print("Either file does not exist or is incompatible") Cam.log += '\nERROR: DNG file does not exist or is incompatible' raise return Img ''' load image from file location and perform calibration check correct filetype mac boolean is true if image is expected to contain macbeth chart and false if not (alsc images don't have macbeth charts) ''' def load_image(Cam, im_str, mac_config=None, show=False, mac=True, show_meta=False): """ check image is correct filetype """ if '.jpg' in im_str or '.jpeg' in im_str or '.brcm' in im_str or '.dng' in im_str: if '.dng' in im_str: Img = dng_load_image(Cam, im_str) else: Img = brcm_load_image(Cam, im_str) """ handle errors smoothly if loading image failed """ if Img == 0: return 0 if show_meta: Img.print_meta() if mac: """ find macbeth centres, discarding images that are too dark or light """ av_chan = (np.mean(np.array(Img.channels), axis=0)/(2**16)) av_val = np.mean(av_chan) # print(av_val) if av_val < Img.blacklevel_16/(2**16)+1/64: macbeth = None print('\nError: Image too dark!') Cam.log += '\nWARNING: Image too dark!' else: macbeth = find_macbeth(Cam, av_chan, mac_config) """ if no macbeth found return error """ if macbeth is None: print('\nERROR: No macbeth chart found') return 0 mac_cen_coords = macbeth[1] # print('\nMacbeth centres located successfully') """ obtain image patches """ # print('\nObtaining image patches') Img.get_patches(mac_cen_coords) if Img.saturated: print('\nERROR: Macbeth patches have saturated') Cam.log += '\nWARNING: Macbeth patches have saturated!' return 0 """ clear memory """ Img.buf = None del Img.buf # print('Image patches obtained successfully') """ optional debug """ if show and __name__ == '__main__': copy = sum(Img.channels)/2**18 copy = np.reshape(copy, (Img.h//2, Img.w//2)).astype(np.float64) copy, _ = reshape(copy, 800) represent(copy) return Img """ return error if incorrect filetype """ else: # print('\nERROR:\nInvalid file extension') return 0 """ bytearray splice to number little endian """ def ba_to_b(b): total = 0 for i in range(len(b)): total += 256**i * b[i] return total