diff options
Diffstat (limited to 'Documentation')
34 files changed, 1566 insertions, 835 deletions
diff --git a/Documentation/Doxyfile.in b/Documentation/Doxyfile-common.in index a86ea6c1..045c19dd 100644 --- a/Documentation/Doxyfile.in +++ b/Documentation/Doxyfile-common.in @@ -20,35 +20,19 @@ TOC_INCLUDE_HEADINGS = 0 CASE_SENSE_NAMES = YES QUIET = YES - -INPUT = "@TOP_SRCDIR@/include/libcamera" \ - "@TOP_SRCDIR@/src/ipa/ipu3" \ - "@TOP_SRCDIR@/src/ipa/libipa" \ - "@TOP_SRCDIR@/src/libcamera" \ - "@TOP_BUILDDIR@/include/libcamera" \ - "@TOP_BUILDDIR@/src/libcamera" +WARN_AS_ERROR = @WARN_AS_ERROR@ FILE_PATTERNS = *.c \ *.cpp \ + *.dox \ *.h RECURSIVE = YES -EXCLUDE = @TOP_SRCDIR@/include/libcamera/base/span.h \ - @TOP_SRCDIR@/include/libcamera/internal/device_enumerator_sysfs.h \ - @TOP_SRCDIR@/include/libcamera/internal/device_enumerator_udev.h \ - @TOP_SRCDIR@/include/libcamera/internal/ipc_pipe_unixsocket.h \ - @TOP_SRCDIR@/src/libcamera/device_enumerator_sysfs.cpp \ - @TOP_SRCDIR@/src/libcamera/device_enumerator_udev.cpp \ - @TOP_SRCDIR@/src/libcamera/ipc_pipe_unixsocket.cpp \ - @TOP_SRCDIR@/src/libcamera/pipeline/ \ - @TOP_SRCDIR@/src/libcamera/tracepoints.cpp \ - @TOP_BUILDDIR@/include/libcamera/internal/tracepoints.h \ - @TOP_BUILDDIR@/src/libcamera/proxy/ - EXCLUDE_PATTERNS = @TOP_BUILDDIR@/include/libcamera/ipa/*_serializer.h \ @TOP_BUILDDIR@/include/libcamera/ipa/*_proxy.h \ @TOP_BUILDDIR@/include/libcamera/ipa/ipu3_*.h \ + @TOP_BUILDDIR@/include/libcamera/ipa/mali-c55_*.h \ @TOP_BUILDDIR@/include/libcamera/ipa/raspberrypi_*.h \ @TOP_BUILDDIR@/include/libcamera/ipa/rkisp1_*.h \ @TOP_BUILDDIR@/include/libcamera/ipa/vimc_*.h @@ -68,8 +52,6 @@ EXCLUDE_SYMBOLS = libcamera::BoundMethodArgs \ EXCLUDE_SYMLINKS = YES -HTML_OUTPUT = api-html - GENERATE_LATEX = NO MACRO_EXPANSION = YES diff --git a/Documentation/Doxyfile-internal.in b/Documentation/Doxyfile-internal.in new file mode 100644 index 00000000..5343bc2b --- /dev/null +++ b/Documentation/Doxyfile-internal.in @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: CC-BY-SA-4.0 + +@INCLUDE_PATH = @TOP_BUILDDIR@/Documentation +@INCLUDE = Doxyfile-common + +HIDE_UNDOC_CLASSES = NO +HIDE_UNDOC_MEMBERS = NO +HTML_OUTPUT = internal-api-html +INTERNAL_DOCS = YES +ENABLED_SECTIONS = internal + +INPUT = "@TOP_SRCDIR@/Documentation" \ + "@TOP_SRCDIR@/include/libcamera" \ + "@TOP_SRCDIR@/src/ipa/ipu3" \ + "@TOP_SRCDIR@/src/ipa/libipa" \ + "@TOP_SRCDIR@/src/libcamera" \ + "@TOP_BUILDDIR@/include/libcamera" \ + "@TOP_BUILDDIR@/src/libcamera" + +EXCLUDE = @TOP_SRCDIR@/include/libcamera/base/span.h \ + @TOP_SRCDIR@/include/libcamera/internal/device_enumerator_sysfs.h \ + @TOP_SRCDIR@/include/libcamera/internal/device_enumerator_udev.h \ + @TOP_SRCDIR@/include/libcamera/internal/ipc_pipe_unixsocket.h \ + @TOP_SRCDIR@/src/libcamera/device_enumerator_sysfs.cpp \ + @TOP_SRCDIR@/src/libcamera/device_enumerator_udev.cpp \ + @TOP_SRCDIR@/src/libcamera/ipc_pipe_unixsocket.cpp \ + @TOP_SRCDIR@/src/libcamera/pipeline/ \ + @TOP_SRCDIR@/src/libcamera/sensor/camera_sensor_legacy.cpp \ + @TOP_SRCDIR@/src/libcamera/sensor/camera_sensor_raw.cpp \ + @TOP_SRCDIR@/src/libcamera/tracepoints.cpp \ + @TOP_BUILDDIR@/include/libcamera/internal/tracepoints.h \ + @TOP_BUILDDIR@/include/libcamera/ipa/soft_ipa_interface.h \ + @TOP_BUILDDIR@/src/libcamera/proxy/ diff --git a/Documentation/Doxyfile-public.in b/Documentation/Doxyfile-public.in new file mode 100644 index 00000000..36bb5758 --- /dev/null +++ b/Documentation/Doxyfile-public.in @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: CC-BY-SA-4.0 + +@INCLUDE_PATH = @TOP_BUILDDIR@/Documentation +@INCLUDE = Doxyfile-common + +HIDE_UNDOC_CLASSES = YES +HIDE_UNDOC_MEMBERS = YES +HTML_OUTPUT = api-html +INTERNAL_DOCS = NO + +INPUT = "@TOP_SRCDIR@/Documentation" \ + ${inputs} + +EXCLUDE = @TOP_SRCDIR@/include/libcamera/base/class.h \ + @TOP_SRCDIR@/include/libcamera/base/object.h \ + @TOP_SRCDIR@/include/libcamera/base/span.h \ + @TOP_SRCDIR@/src/libcamera/base/class.cpp \ + @TOP_SRCDIR@/src/libcamera/base/object.cpp + +PREDEFINED += __DOXYGEN_PUBLIC__ diff --git a/Documentation/api-html/index.rst b/Documentation/api-html/index.rst index 9e630fc0..2f09833d 100644 --- a/Documentation/api-html/index.rst +++ b/Documentation/api-html/index.rst @@ -2,7 +2,7 @@ .. _api: -API -=== +API Reference +============= :: Placeholder for Doxygen documentation diff --git a/Documentation/camera-sensor-model.rst b/Documentation/camera-sensor-model.rst index 58bf70e1..87a25bf4 100644 --- a/Documentation/camera-sensor-model.rst +++ b/Documentation/camera-sensor-model.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: documentation-contents.rst + .. _camera-sensor-model: .. todo: Move to Doxygen-generated documentation @@ -105,35 +107,34 @@ control: will be downscaled in its vertical and horizontal sizes by the specified factor. - .. code-block:: + .. code-block:: c :caption: Definition: The horizontal and vertical binning factors horizontal_binning = xBin; vertical_binning = yBin; - - skipping Skipping reduces the image resolution by skipping the read-out of a number of adjacent pixels. The skipping factor is specified by the 'increment' number (number of pixels to 'skip') in the vertical and horizontal directions and for even and odd rows and columns. - .. code-block:: + .. code-block:: c :caption: Definition: The horizontal and vertical skipping factors - horizontal_skipping = (xOddInc + xEvenInc) / 2 - vertical_skipping = (yOddInc + yEvenInc) / 2 + horizontal_skipping = (xOddInc + xEvenInc) / 2; + vertical_skipping = (yOddInc + yEvenInc) / 2; Different sensors perform the binning and skipping stages in different orders. For the sake of computing the final output image size the order of execution is not relevant. The overall down-scaling factor is obtained by combining the binning and skipping factors. - .. code-block:: + .. code-block:: c :caption: Definition: The total scaling factor (binning + sub-sampling) - total_horizontal_downscale = horizontal_binning + horizontal_skipping - total_vertical_downscale = vertical_binning + vertical_skipping + total_horizontal_downscale = horizontal_binning + horizontal_skipping; + total_vertical_downscale = vertical_binning + vertical_skipping; 4. The output size is used to specify any additional cropping on the sub-sampled @@ -159,16 +160,16 @@ configurations: the *pixel rate* of the data sent on the MIPI CSI-2 bus allows to compute the image stream frame rate. The equation is the well known: - .. code-block:: + .. code-block:: c - frame_duration = total_frame_size / pixel_rate - frame_rate = 1 / frame_duration + frame_duration = total_frame_size / pixel_rate; + frame_rate = 1 / frame_duration; where the *pixel_rate* parameter is the result of the sensor's configuration of the MIPI CSI-2 bus *(the following formula applies to MIPI CSI-2 when used on MIPI D-PHY physical protocol layer only)* - .. code-block:: + .. code-block:: c - pixel_rate = CSI-2_link_freq * 2 * nr_of_lanes / bits_per_sample + pixel_rate = csi_2_link_freq * 2 * nr_of_lanes / bits_per_sample; diff --git a/Documentation/code-of-conduct.rst b/Documentation/code-of-conduct.rst new file mode 100644 index 00000000..0edd1e99 --- /dev/null +++ b/Documentation/code-of-conduct.rst @@ -0,0 +1,96 @@ +.. SPDX-License-Identifier: CC-BY-4.0 + +.. include:: documentation-contents.rst + +.. _code-of-conduct: + +Contributor Covenant Code of Conduct +==================================== + +Our Pledge +---------- + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +Our Standards +------------- + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +Our Responsibilities +-------------------- + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +Scope +----- + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +Enforcement +----------- + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at conduct@libcamera.org, or directly to +any member of the code of conduct team: + +* Kieran Bingham <kieran.bingham@ideasonboard.com> +* Laurent Pinchart <laurent.pinchart@ideasonboard.com> + +All complaints will be reviewed and investigated and will result in a response +that is deemed necessary and appropriate to the circumstances. The project team +is obligated to maintain confidentiality with regard to the reporter of an +incident. Further details of specific enforcement policies may be posted +separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +Attribution +----------- + +This Code of Conduct is adapted from the `Contributor Covenant`_, version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +.. _Contributor Covenant: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq + diff --git a/Documentation/coding-style.rst b/Documentation/coding-style.rst index 053fdd99..6ac3a4a0 100644 --- a/Documentation/coding-style.rst +++ b/Documentation/coding-style.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: documentation-contents.rst + .. _coding-style-guidelines: Coding Style Guidelines @@ -59,7 +61,7 @@ document: underscores in between * All formatting rules specified in the selected sections of the Linux kernel Code Style for indentation, braces, spacing, etc -* Header guards are formatted as '__LIBCAMERA_FILE_NAME_H__' +* Headers are guarded by the use of '#pragma once' Order of Includes ~~~~~~~~~~~~~~~~~ @@ -215,7 +217,7 @@ shall be avoided when possible, but are allowed when required (for instance to implement factories with auto-registration). They shall not depend on any other global variable, should run a minimal amount of code in the constructor and destructor, and code that contains dependencies should be moved to a later -point in time. +point in time. Error Handling ~~~~~~~~~~~~~~ diff --git a/Documentation/conf.py b/Documentation/conf.py index 7eeea7f3..089f114c 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -37,8 +37,11 @@ author = u'Kieran Bingham, Jacopo Mondi, Laurent Pinchart, Niklas Söderlund' # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinx.ext.graphviz' ] +graphviz_output_format = 'svg' + # Add any paths that contain templates here, relative to this directory. templates_path = [] @@ -61,7 +64,12 @@ language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = [ + '_build', + 'Thumbs.db', + '.DS_Store', + 'documentation-contents.rst', +] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None diff --git a/Documentation/contributing.rst b/Documentation/contributing.rst index 2f0b4921..18b1914a 100644 --- a/Documentation/contributing.rst +++ b/Documentation/contributing.rst @@ -8,6 +8,10 @@ Whether you would like to help with coding, documentation, testing, proposing new features, or just discussing the project with the community, you can join our official public communication channels, or simply check out the code. +The project adheres to a :ref:`code of conduct <code-of-conduct>` that +maintainers, contributors and community members are expected to follow in all +online and offline communication. + Mailing List ------------ @@ -134,4 +138,5 @@ By making a contribution to this project, I certify that: .. toctree:: :hidden: + Code of Conduct <code-of-conduct> Coding Style <coding-style> diff --git a/Documentation/design/ae.rst b/Documentation/design/ae.rst new file mode 100644 index 00000000..df9b1fa7 --- /dev/null +++ b/Documentation/design/ae.rst @@ -0,0 +1,331 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +Design of Exposure and Gain controls +==================================== + +This document explains the design and rationale of the controls related to +exposure and gain. This includes the all-encompassing auto-exposure (AE), the +manual exposure control, and the manual gain control. + +Description of the problem +-------------------------- + +Sub controls +^^^^^^^^^^^^ + +There are more than one control that make up total exposure: exposure time, +gain, and aperture (though for now we will not consider aperture). We already +had individual controls for setting the values of manual exposure and manual +gain, but for switching between auto mode and manual mode we only had a +high-level boolean AeEnable control that would set *both* exposure and gain to +auto mode or manual mode; we had no way to set one to auto and the other to +manual. + +So, we need to introduce two new controls to act as "levers" to indicate +individually for exposure and gain if the value would come from AEGC or if it +would come from the manual control value. + +Aperture priority +^^^^^^^^^^^^^^^^^ + +We eventually may need to support aperture, and so whatever our solution is for +having only some controls on auto and the others on manual needs to be +extensible. + +Flickering when going from auto to manual +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When a manual exposure or gain value is requested by the application, it costs +a few frames worth of time for them to take effect. This means that during a +transition from auto to manual, there would be flickering in the control values +and the transition won't be smooth. + +Take for instance the following flow, where we start on auto exposure (which +for the purposes of the example increments by 1 each frame) and we want to +switch seamlessly to manual exposure, which involves copying the exposure value +computed by the auto exposure algorithm: + +:: + + +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ + | N | | N+1 | | N+2 | | N+3 | | N+4 | | N+5 | | N+6 | + +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ + + Mode requested: Auto Auto Auto Manual Manual Manual Manual + Exp requested: N/A N/A N/A 2 2 2 2 + Set in Frame: N+2 N+3 N+4 N+5 N+6 N+7 N+8 + + Mode used: Auto Auto Auto Auto Auto Manual Manual + Exp used: 0 1 2 3 4 2 2 + +As we can see, after frame N+2 completes, we copy the exposure value that was +used for frame N+2 (which was computed by AE algorithm), and queue that value +into request N+3 with manual mode on. However, as it takes two frames for the +exposure to be set, the exposure still changes since it is set by AE, and we +get a flicker in the exposure during the switch from auto to manual. + +A solution is to *not submit* any exposure value when manual mode is enabled, +and wait until the manual mode as been "applied" before copying the exposure +value: + +:: + + +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ + | N | | N+1 | | N+2 | | N+3 | | N+4 | | N+5 | | N+6 | + +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ + + Mode requested: Auto Auto Auto Manual Manual Manual Manual + Exp requested: N/A N/A N/A None None None 5 + Set in Frame: N+2 N+3 N+4 N+5 N+6 N+7 N+8 + + Mode used: Auto Auto Auto Auto Auto Manual Manual + Exp used: 0 1 2 3 4 5 5 + +In practice, this works. However, libcamera has a policy where once a control +is submitted, its value is saved and does not need to be resubmitted. If the +manual exposure value was set while auto mode was on, in theory the value would +be saved, so when manual mode is enabled, the exposure value that was +previously set would immediately be used. Clearly this solution isn't correct, +but it can serve as the basis for a proper solution, with some more rigorous +rules. + +Existing solutions +------------------ + +Raspberry Pi +^^^^^^^^^^^^ + +The Raspberry Pi IPA gets around the lack of individual AeEnable controls for +exposure and gain by using magic values. When AeEnable is false, if one of the +manual control values was set to 0 then the value computed by AEGC would be +used for just that control. This solution isn't desirable, as it prevents +that magic value from being used as a valid value. + +To get around the flickering issue, when AeEnable is false, the Raspberry Pi +AEGC simply stops updating the values to be set, without restoring the +previously set manual exposure time and gain. This works, but is not a proper +solution. + +Android +^^^^^^^ + +The Android HAL specification requires that exposure and gain (sensitivity) +must both be manual or both be auto. It cannot be that one is manual while the +other is auto, so they simply don't support sub controls. + +For the flickering issue, the Android HAL has an AeLock control. To transition +from auto to manual, the application would keep AE on auto, and turn on the +lock. Once the lock has propagated through, then the value can be copied from +the result into the request and the lock disabled and the mode set to manual. + +The problem with this solution is, besides the extra complexity, that it is +ambiguous what happens if there is a state transition from manual to locked +(even though it's a state transition that doesn't make sense). If locked is +defined to "use the last automatically computed values" then it could use the +values from the last time it AE was set to auto, or it would be undefined if AE +was never auto (eg. it started out as manual), or if AE is implemented to run +in the background it could just use the current values that are computed. If +locked is defined to "use the last value that was set" there would be less +ambiguity. Still, it's better if we can make it impossible to execute this +nonsensical state transition, and if we can reduce the complexity of having +this extra control or extra setting on a lever. + +Summary of goals +---------------- + +- We need a lock of some sort, to instruct the AEGC to not update output + results + +- We need manual modes, to override the values computed by the AEGC + +- We need to support seamless transitions from auto to manual, and do so + without flickering + +- We need custom minimum values for the manual controls; that is, no magic + values for enabling/disabling auto + +- All of these need to be done with AE sub-controls (exposure time, analogue + gain) and be extensible to aperture in the future + +Our solution +------------ + +A diagram of our solution: + +:: + + +----------------------------+-------------+------------------+-----------------+ + | INPUT | ALGORITHM | RESULT | OUTPUT | + +----------------------------+-------------+------------------+-----------------+ + + ExposureTimeMode ExposureTimeMode + ---------------------+----------------------------------------+-----------------> + 0: Auto | | + 1: Manual | V + | |\ + | | \ + | /----------------------------------> | 1| ExposureTime + | | +-------------+ exposure time | | --------------> + \--)--> | | --------------> | 0| + ExposureTime | | | | / + ------------------------+--> | | |/ + | | AeState + | AEGC | -----------------------------------> + AnalogueGain | | + ------------------------+--> | | |\ + | | | | \ + /--)--> | | --------------> | 0| AnalogueGain + | | +-------------+ analogue gain | | --------------> + | \----------------------------------> | 1| + | | / + | |/ + | ^ + AnalogueGainMode | | AnalogueGainMode + ---------------------+----------------------------------------+-----------------> + 0: Auto + 1: Manual + + AeEnable + - True -> ExposureTimeMode:Auto + AnalogueGainMode:Auto + - False -> ExposureTimeMode:Manual + AnalogueGainMode:Manual + + +The diagram is divided in four sections horizontally: + +- Input: The values received from the request controls + +- Algorithm: The algorithm itself + +- Result: The values calculated by the algorithm + +- Output: The values reported in result metadata and applied to the device + +The four input controls are divided between manual values (ExposureTime and +AnalogueGain), and operation modes (ExposureTimeMode and AnalogueGainMode). The +former are the manual values, the latter control how they're applied. The two +modes are independent from each other, and each can take one of two values: + +- Auto (0): The AGC computes the value normally. The AGC result is applied + to the output. The manual value is ignored *and is not retained*. + +- Manual (1): The AGC uses the manual value internally. The corresponding + manual control from the request is applied to the output. The AGC result + is ignored. + +The AeState control reports the state of the unified AEGC block. If both +ExposureTimeMode and AnalogueGainMode are set to manual then it will report +Idle. If at least one of the two is set to auto, then AeState will report +if the AEGC has Converged or not (Searching). This control replaces the old +AeLocked control, as it was insufficient for reporting the AE state. + +There is a caveat to manual mode: the manual control value is not retained if +it is set during auto mode. This means that if manual mode is entered without +also setting the manual value, then it will enter a state similar to "locked", +where the last automatically computed value while the mode was auto will be +used. Once the manual value is set, then that will be used and retained as +usual. + +This simulates an auto -> locked -> manual or auto -> manual state transition, +and makes it impossible to do the nonsensical manual -> locked state +transition. + +AeEnable still exists to allow applications to set the mode of all the +sub-controls at once. Besides being for convenience, this will also be useful +when we eventually implement an aperture control. This is because applications +that will be made before aperture will have been available would still be able +to set aperture mode to auto or manual, as opposed to having the aperture stuck +at auto while the application really wanted manual. Although the aperture would +still be stuck at an uncontrollable value, at least it would be at a static +usable value as opposed to varying via the AEGC algorithm. + +With this solution, the earlier example would become: + +:: + + +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ + | N+2 | | N+3 | | N+4 | | N+5 | | N+6 | | N+7 | | N+8 | | N+9 | | N+10| + +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ +-----+ + Mode requested: Auto Manual Manual Manual Manual Manual Manual Manual Manual + Exp requested: N/A None None None None 10 None 10 10 + Set in Frame: N+4 N+5 N+6 N+7 N+8 N+9 N+10 N+11 N+12 + + Mode used: Auto Auto Auto Manual Manual Manual Manual Manual Manual + Exp used: 2 3 4 5 5 5 5 10 10 + +This example is extended by a few frames to exhibit the simulated "locked" +state. At frame N+5 the application has confirmed that the manual mode has been +entered, but does not provide a manual value until request N+7. Thus, the value +that is used in requests N+5 and N+6 (where the mode is disabled), comes from +the last value that was used when the mode was auto, which comes from frame +N+4. + +Then, in N+7, a manual value of 10 is supplied. It takes until frame N+9 for +the exposure to be applied. N+8 does not supply a manual value, but the last +supplied value is retained, so a manual value of 10 is still used and set in +frame N+10. + +Although this behavior is the same as what we had with waiting for the manual +mode to propagate (in the section "Description of the problem"), this time it +is correct as we have defined specifically that if a manual value was specified +while the mode was auto, it will not be retained. + +Description of the controls +--------------------------- + +As described above, libcamera offers the following controls related to exposure +and gain: + +- AnalogueGain + +- AnalogueGainMode + +- ExposureTime + +- ExposureTimeMode + +- AeState + +- AeEnable + +Auto-exposure and auto-gain can be enabled and disabled separately using the +ExposureTimeMode and AnalogueGainMode controls respectively. The AeEnable +control can also be used, as it sets both of the modes simultaneously. The +AeEnable control is not returned in metadata. + +When the respective mode is set to auto, the respective value that is computed +by the AEGC algorithm is applied to the image sensor. Any value that is +supplied in the manual ExposureTime/AnalogueGain control is ignored and not +retained. Another way to understand this is that when the mode transitions from +auto to manual, the internally stored control value is overwritten with the +last value computed by the auto algorithm. + +This means that when we transition from auto to manual without supplying a +manual control value, the last value that was set by the AEGC algorithm will +keep be used. This can be used to do a flickerless transition from auto to +manual as described earlier. If the camera started out in manual mode and no +corresponding value has been supplied yet, then a best-effort default value +shall be set. + +The manual control value can be set in the same request as setting the mode to +auto if the desired manual control value is already known. + +Transitioning from manual to auto shall be implicitly flickerless, as the AEGC +algorithms are expected to start running from the last manual value. + +The AeState metadata reports the state of the AE algorithm. As AE cannot +compute exposure and gain separately, the state of the AE component is +unified. There are three states: Idle, Searching, and Converged. + +The state shall be Idle if both ExposureTimeMode and AnalogueGainMode +are set to Manual. If the camera only supports one of the two controls, +then the state shall be Idle if that one control is set to Manual. If +the camera does not support Manual for at least one of the two controls, +then the state will never be Idle, as AE will always be running. + +The state shall be Searching if at least one of exposure or gain calculated +by the AE algorithm is used (that is, at least one of the two modes is Auto), +*and* the value(s) have not converged yet. + +The state shall be Converged if at least one of exposure or gain calculated +by the AE algorithm is used (that is, at least one of the two modes is Auto), +*and* the value(s) have converged. diff --git a/Documentation/docs.rst b/Documentation/docs.rst deleted file mode 100644 index a6e8a59a..00000000 --- a/Documentation/docs.rst +++ /dev/null @@ -1,400 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-SA-4.0 - -.. contents:: - :local: - -************* -Documentation -************* - -.. toctree:: - :hidden: - - API <api-html/index> - -API -=== - -The libcamera API is extensively documented using Doxygen. The :ref:`API -nightly build <api>` contains the most up-to-date API documentation, built from -the latest master branch. - -Feature Requirements -==================== - -Device enumeration ------------------- - -The library shall support enumerating all camera devices available in the -system, including both fixed cameras and hotpluggable cameras. It shall -support cameras plugged and unplugged after the initialization of the -library, and shall offer a mechanism to notify applications of camera plug -and unplug. - -The following types of cameras shall be supported: - -* Internal cameras designed for point-and-shoot still image and video - capture usage, either controlled directly by the CPU, or exposed through - an internal USB bus as a UVC device. - -* External UVC cameras designed for video conferencing usage. - -Other types of camera, including analog cameras, depth cameras, thermal -cameras, external digital picture or movie cameras, are out of scope for -this project. - -A hardware device that includes independent camera sensors, such as front -and back sensors in a phone, shall be considered as multiple camera devices -for the purpose of this library. - -Independent Camera Devices --------------------------- - -When multiple cameras are present in the system and are able to operate -independently from each other, the library shall expose them as multiple -camera devices and support parallel operation without any additional usage -restriction apart from the limitations inherent to the hardware (such as -memory bandwidth, CPU usage or number of CSI-2 receivers for instance). - -Independent processes shall be able to use independent cameras devices -without interfering with each other. A single camera device shall be -usable by a single process at a time. - -Multiple streams support ------------------------- - -The library shall support multiple video streams running in parallel -for each camera device, within the limits imposed by the system. - -Per frame controls ------------------- - -The library shall support controlling capture parameters for each stream -on a per-frame basis, on a best effort basis based on the capabilities of the -hardware and underlying software stack (including kernel drivers and -firmware). It shall apply capture parameters to the frame they target, and -report the value of the parameters that have effectively been used for each -captured frame. - -When a camera device supports multiple streams, the library shall allow both -control of each stream independently, and control of multiple streams -together. Streams that are controlled together shall be synchronized. No -synchronization is required for streams controlled independently. - -Capability Enumeration ----------------------- - -The library shall expose capabilities of each camera device in a way that -allows applications to discover those capabilities dynamically. Applications -shall be allowed to cache capabilities for as long as they are using the -library. If capabilities can change at runtime, the library shall offer a -mechanism to notify applications of such changes. Applications shall not -cache capabilities in long term storage between runs. - -Capabilities shall be discovered dynamically at runtime from the device when -possible, and may come, in part or in full, from platform configuration -data. - -Device Profiles ---------------- - -The library may define different camera device profiles, each with a minimum -set of required capabilities. Applications may use those profiles to quickly -determine the level of features exposed by a device without parsing the full -list of capabilities. Camera devices may implement additional capabilities -on top of the minimum required set for the profile they expose. - -3A and Image Enhancement Algorithms ------------------------------------ - -The camera devices shall implement auto exposure, auto gain and auto white -balance. Camera devices that include a focus lens shall implement auto -focus. Additional image enhancement algorithms, such as noise reduction or -video stabilization, may be implemented. - -All algorithms may be implemented in hardware or firmware outside of the -library, or in software in the library. They shall all be controllable by -applications. - -The library shall be architectured to isolate the 3A and image enhancement -algorithms in a component with a documented API, respectively called the 3A -component and the 3A API. The 3A API shall be stable, and shall allow both -open-source and closed-source implementations of the 3A component. - -The library may include statically-linked open-source 3A components, and -shall support dynamically-linked open-source and closed-source 3A -components. - -Closed-source 3A Component Sandboxing -------------------------------------- - -For security purposes, it may be desired to run closed-source 3A components -in a separate process. The 3A API would in such a case be transported over -IPC. The 3A API shall make it possible to use any IPC mechanism that -supports passing file descriptors. - -The library may implement an IPC mechanism, and shall support third-party -platform-specific IPC mechanisms through the implementation of a -platform-specific 3A API wrapper. No modification to the library shall be -needed to use such third-party IPC mechanisms. - -The 3A component shall not directly access any device node on the system. -Such accesses shall instead be performed through the 3A API. The library -shall validate all accesses and restrict them to what is absolutely required -by 3A components. - -V4L2 Compatibility Layer ------------------------- - -The project shall support traditional V4L2 application through an additional -libcamera wrapper library. The wrapper library shall trap all accesses to -camera devices through `LD_PRELOAD`, and route them through libcamera to -emulate a high-level V4L2 camera device. It shall expose camera device -features on a best-effort basis, and aim for the level of features -traditionally available from a UVC camera designed for video conferencing. - -Android Camera HAL v3 Compatibility ------------------------------------ - -The library API shall expose all the features required to implement an -Android Camera HAL v3 on top of libcamera. Some features of the HAL may be -omitted as long as they can be implemented separately in the HAL, such as -JPEG encoding, or YUV reprocessing. - - -Camera Stack -============ - -:: - - a c / +-------------+ +-------------+ +-------------+ +-------------+ - p a | | Native | | Framework | | Native | | Android | - p t | | V4L2 | | Application | | libcamera | | Camera | - l i | | Application | | (gstreamer) | | Application | | Framework | - i o \ +-------------+ +-------------+ +-------------+ +-------------+ - n ^ ^ ^ ^ - | | | | - l a | | | | - i d v v | v - b a / +-------------+ +-------------+ | +-------------+ - c p | | V4L2 | | Camera | | | Android | - a t | | Compat. | | Framework | | | Camera | - m a | | | | (gstreamer) | | | HAL | - e t \ +-------------+ +-------------+ | +-------------+ - r i ^ ^ | ^ - a o | | | | - n | | | | - / | ,................................................ - | | ! : Language : ! - l f | | ! : Bindings : ! - i r | | ! : (optional) : ! - b a | | \...............................................' - c m | | | | | - a e | | | | | - m w | v v v v - e o | +----------------------------------------------------------------+ - r r | | | - a k | | libcamera | - | | | - \ +----------------------------------------------------------------+ - ^ ^ ^ - Userspace | | | - ------------------------ | ---------------- | ---------------- | --------------- - Kernel | | | - v v v - +-----------+ +-----------+ +-----------+ - | Media | <--> | Video | <--> | V4L2 | - | Device | | Device | | Subdev | - +-----------+ +-----------+ +-----------+ - -The camera stack comprises four software layers. From bottom to top: - -* The kernel drivers control the camera hardware and expose a - low-level interface to userspace through the Linux kernel V4L2 - family of APIs (Media Controller API, V4L2 Video Device API and - V4L2 Subdev API). - -* The libcamera framework is the core part of the stack. It - handles all control of the camera devices in its core component, - libcamera, and exposes a native C++ API to upper layers. Optional - language bindings allow interfacing to libcamera from other - programming languages. - - Those components live in the same source code repository and - all together constitute the libcamera framework. - -* The libcamera adaptation is an umbrella term designating the - components that interface to libcamera in other frameworks. - Notable examples are a V4L2 compatibility layer, a gstreamer - libcamera element, and an Android camera HAL implementation based - on libcamera. - - Those components can live in the libcamera project source code - in separate repositories, or move to their respective project's - repository (for instance the gstreamer libcamera element). - -* The applications and upper level frameworks are based on the - libcamera framework or libcamera adaptation, and are outside of - the scope of the libcamera project. - - -libcamera Architecture -====================== - -:: - - ---------------------------< libcamera Public API >--------------------------- - ^ ^ - | | - v v - +-------------+ +-------------------------------------------------+ - | Camera | | Camera Device | - | Devices | | +---------------------------------------------+ | - | Manager | | | Device-Agnostic | | - +-------------+ | | | | - ^ | | +------------------------+ | - | | | | ~~~~~~~~~~~~~~~~~~~~~ | - | | | | { +---------------+ } | - | | | | } | ////Image//// | { | - | | | | <-> | /Processing// | } | - | | | | } | /Algorithms// | { | - | | | | { +---------------+ } | - | | | | ~~~~~~~~~~~~~~~~~~~~~ | - | | | | ======================== | - | | | | +---------------+ | - | | | | | //Pipeline/// | | - | | | | <-> | ///Handler/// | | - | | | | | ///////////// | | - | | +--------------------+ +---------------+ | - | | Device-Specific | - | +-------------------------------------------------+ - | ^ ^ - | | | - v v v - +--------------------------------------------------------------------+ - | Helpers and Support Classes | - | +-------------+ +-------------+ +-------------+ +-------------+ | - | | MC & V4L2 | | Buffers | | Sandboxing | | Plugins | | - | | Support | | Allocator | | IPC | | Manager | | - | +-------------+ +-------------+ +-------------+ +-------------+ | - | +-------------+ +-------------+ | - | | Pipeline | | ... | | - | | Runner | | | | - | +-------------+ +-------------+ | - +--------------------------------------------------------------------+ - - /// Device-Specific Components - ~~~ Sandboxing - -While offering a unified API towards upper layers, and presenting -itself as a single library, libcamera isn't monolithic. It exposes -multiple components through its public API, is built around a set of -separate helpers internally, uses device-specific components and can -load dynamic plugins. - -Camera Devices Manager - The Camera Devices Manager provides a view of available cameras - in the system. It performs cold enumeration and runtime camera - management, and supports a hotplug notification mechanism in its - public API. - - To avoid the cost associated with cold enumeration of all devices - at application start, and to arbitrate concurrent access to camera - devices, the Camera Devices Manager could later be split to a - separate service, possibly with integration in platform-specific - device management. - -Camera Device - The Camera Device represents a camera device to upper layers. It - exposes full control of the device through the public API, and is - thus the highest level object exposed by libcamera. - - Camera Device instances are created by the Camera Devices - Manager. An optional function to create new instances could be exposed - through the public API to speed up initialization when the upper - layer knows how to directly address camera devices present in the - system. - -Pipeline Handler - The Pipeline Handler manages complex pipelines exposed by the kernel drivers - through the Media Controller and V4L2 APIs. It abstracts pipeline handling to - hide device-specific details to the rest of the library, and implements both - pipeline configuration based on stream configuration, and pipeline runtime - execution and scheduling when needed by the device. - - This component is device-specific and is part of the libcamera code base. As - such it is covered by the same free software license as the rest of libcamera - and needs to be contributed upstream by device vendors. The Pipeline Handler - lives in the same process as the rest of the library, and has access to all - helpers and kernel camera-related devices. - -Image Processing Algorithms - Together with the hardware image processing and hardware statistics - collection, the Image Processing Algorithms implement 3A (Auto-Exposure, - Auto-White Balance and Auto-Focus) and other algorithms. They run on the CPU - and interact with the kernel camera devices to control hardware image - processing based on the parameters supplied by upper layers, closing the - control loop of the ISP. - - This component is device-specific and is loaded as an external plugin. It can - be part of the libcamera code base, in which case it is covered by the same - license, or provided externally as an open-source or closed-source component. - - The component is sandboxed and can only interact with libcamera through - internal APIs specifically marked as such. In particular it will have no - direct access to kernel camera devices, and all its accesses to image and - metadata will be mediated by dmabuf instances explicitly passed to the - component. The component must be prepared to run in a process separate from - the main libcamera process, and to have a very restricted view of the system, - including no access to networking APIs and limited access to file systems. - - The sandboxing mechanism isn't defined by libcamera. One example - implementation will be provided as part of the project, and platforms vendors - will be able to provide their own sandboxing mechanism as a plugin. - - libcamera should provide a basic implementation of Image Processing - Algorithms, to serve as a reference for the internal API. Device vendors are - expected to provide a full-fledged implementation compatible with their - Pipeline Handler. One goal of the libcamera project is to create an - environment in which the community will be able to compete with the - closed-source vendor binaries and develop a high quality open source - implementation. - -Helpers and Support Classes - While Pipeline Handlers are device-specific, implementations are expected to - share code due to usage of identical APIs towards the kernel camera drivers - and the Image Processing Algorithms. This includes without limitation handling - of the MC and V4L2 APIs, buffer management through dmabuf, and pipeline - discovery, configuration and scheduling. Such code will be factored out to - helpers when applicable. - - Other parts of libcamera will also benefit from factoring code out to - self-contained support classes, even if such code is present only once in the - code base, in order to keep the source code clean and easy to read. This - should be the case for instance for plugin management. - - -V4L2 Compatibility Layer ------------------------- - -V4L2 compatibility is achieved through a shared library that traps all -accesses to camera devices and routes them to libcamera to emulate high-level -V4L2 camera devices. It is injected in a process address space through -`LD_PRELOAD` and is completely transparent for applications. - -The compatibility layer exposes camera device features on a best-effort basis, -and aims for the level of features traditionally available from a UVC camera -designed for video conferencing. - - -Android Camera HAL ------------------- - -Camera support for Android is achieved through a generic Android -camera HAL implementation on top of libcamera. The HAL will implement internally -features required by Android and missing from libcamera, such as JPEG encoding -support. - -The Android camera HAL implementation will initially target the -LIMITED hardware level, with support for the FULL level then being gradually -implemented. diff --git a/Documentation/documentation-contents.rst b/Documentation/documentation-contents.rst new file mode 100644 index 00000000..5c111849 --- /dev/null +++ b/Documentation/documentation-contents.rst @@ -0,0 +1,35 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +.. container:: documentation-nav + + * **Documentation for Users** + * :doc:`Introduction </introduction>` + * :doc:`/feature_requirements` + * :doc:`/guides/application-developer` + * :doc:`/python-bindings` + * :doc:`/environment_variables` + * :doc:`/api-html/index` + * :doc:`/code-of-conduct` + * | + * **Documentation for Developers** + * :doc:`/libcamera_architecture` + * :doc:`/guides/pipeline-handler` + * :doc:`/guides/ipa` + * :doc:`/camera-sensor-model` + * :doc:`/guides/tracing` + * :doc:`/software-isp-benchmarking` + * :doc:`/coding-style` + * :doc:`/internal-api-html/index` + * | + * **Documentation for System Integrators** + * :doc:`/lens_driver_requirements` + * :doc:`/sensor_driver_requirements` + +.. + The following directive adds the "documentation" class to all of the pages + generated by sphinx. This is not relevant in libcamera nor addressed in the + theme's CSS, since all of the pages here are documentation. It **is** used + to properly format the documentation pages on libcamera.org and so should not + be removed. + +.. rst-class:: documentation diff --git a/Documentation/environment_variables.rst b/Documentation/environment_variables.rst index a9b230bc..6f123558 100644 --- a/Documentation/environment_variables.rst +++ b/Documentation/environment_variables.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: documentation-contents.rst + Environment variables ===================== @@ -37,11 +39,29 @@ LIBCAMERA_IPA_MODULE_PATH Example value: ``${HOME}/.libcamera/lib:/opt/libcamera/vendor/lib`` +LIBCAMERA_IPA_PROXY_PATH + Define custom full path for a proxy worker for a given executable name. + + Example value: ``${HOME}/.libcamera/proxy/worker:/opt/libcamera/vendor/proxy/worker`` + +LIBCAMERA_PIPELINES_MATCH_LIST + Define an ordered list of pipeline names to be used to match the media + devices in the system. The pipeline handler names used to populate the + variable are the ones passed to the REGISTER_PIPELINE_HANDLER() macro in the + source code. + + Example value: ``rkisp1,simple`` + LIBCAMERA_RPI_CONFIG_FILE Define a custom configuration file to use in the Raspberry Pi pipeline handler. Example value: ``/usr/local/share/libcamera/pipeline/rpi/vc4/minimal_mem.yaml`` +LIBCAMERA_<NAME>_TUNING_FILE + Define a custom IPA tuning file to use with the pipeline handler `NAME`. + + Example value: ``/usr/local/share/libcamera/ipa/rpi/vc4/custom_sensor.json`` + Further details --------------- diff --git a/Documentation/feature_requirements.rst b/Documentation/feature_requirements.rst new file mode 100644 index 00000000..e6b74a62 --- /dev/null +++ b/Documentation/feature_requirements.rst @@ -0,0 +1,150 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +.. include:: documentation-contents.rst + +Feature Requirements +==================== + +Device enumeration +------------------ + +The library shall support enumerating all camera devices available in the +system, including both fixed cameras and hotpluggable cameras. It shall +support cameras plugged and unplugged after the initialization of the +library, and shall offer a mechanism to notify applications of camera plug +and unplug. + +The following types of cameras shall be supported: + +* Internal cameras designed for point-and-shoot still image and video + capture usage, either controlled directly by the CPU, or exposed through + an internal USB bus as a UVC device. + +* External UVC cameras designed for video conferencing usage. + +Other types of camera, including analog cameras, depth cameras, thermal +cameras, external digital picture or movie cameras, are out of scope for +this project. + +A hardware device that includes independent camera sensors, such as front +and back sensors in a phone, shall be considered as multiple camera devices +for the purpose of this library. + +Independent Camera Devices +-------------------------- + +When multiple cameras are present in the system and are able to operate +independently from each other, the library shall expose them as multiple +camera devices and support parallel operation without any additional usage +restriction apart from the limitations inherent to the hardware (such as +memory bandwidth, CPU usage or number of CSI-2 receivers for instance). + +Independent processes shall be able to use independent cameras devices +without interfering with each other. A single camera device shall be +usable by a single process at a time. + +Multiple streams support +------------------------ + +The library shall support multiple video streams running in parallel +for each camera device, within the limits imposed by the system. + +Per frame controls +------------------ + +The library shall support controlling capture parameters for each stream +on a per-frame basis, on a best effort basis based on the capabilities of the +hardware and underlying software stack (including kernel drivers and +firmware). It shall apply capture parameters to the frame they target, and +report the value of the parameters that have effectively been used for each +captured frame. + +When a camera device supports multiple streams, the library shall allow both +control of each stream independently, and control of multiple streams +together. Streams that are controlled together shall be synchronized. No +synchronization is required for streams controlled independently. + +Capability Enumeration +---------------------- + +The library shall expose capabilities of each camera device in a way that +allows applications to discover those capabilities dynamically. Applications +shall be allowed to cache capabilities for as long as they are using the +library. If capabilities can change at runtime, the library shall offer a +mechanism to notify applications of such changes. Applications shall not +cache capabilities in long term storage between runs. + +Capabilities shall be discovered dynamically at runtime from the device when +possible, and may come, in part or in full, from platform configuration +data. + +Device Profiles +--------------- + +The library may define different camera device profiles, each with a minimum +set of required capabilities. Applications may use those profiles to quickly +determine the level of features exposed by a device without parsing the full +list of capabilities. Camera devices may implement additional capabilities +on top of the minimum required set for the profile they expose. + +3A and Image Enhancement Algorithms +----------------------------------- + +The library shall provide a basic implementation of Image Processing Algorithms +to serve as a reference for the internal API. This shall including auto exposure +and gain and auto white balance. Camera devices that include a focus lens shall +implement auto focus. Additional image enhancement algorithms, such as noise +reduction or video stabilization, may be implemented. Device vendors are +expected to provide a fully-fledged implementation compatible with their +Pipeline Handler. One goal of the libcamera project is to create an environment +in which the community will be able to compete with the closed-source vendor +biaries and develop a high quality open source implementation. + +All algorithms may be implemented in hardware or firmware outside of the +library, or in software in the library. They shall all be controllable by +applications. + +The library shall be architectured to isolate the 3A and image enhancement +algorithms in a component with a documented API, respectively called the 3A +component and the 3A API. The 3A API shall be stable, and shall allow both +open-source and closed-source implementations of the 3A component. + +The library may include statically-linked open-source 3A components, and +shall support dynamically-linked open-source and closed-source 3A +components. + +Closed-source 3A Component Sandboxing +------------------------------------- + +For security purposes, it may be desired to run closed-source 3A components +in a separate process. The 3A API would in such a case be transported over +IPC. The 3A API shall make it possible to use any IPC mechanism that +supports passing file descriptors. + +The library may implement an IPC mechanism, and shall support third-party +platform-specific IPC mechanisms through the implementation of a +platform-specific 3A API wrapper. No modification to the library shall be +needed to use such third-party IPC mechanisms. + +The 3A component shall not directly access any device node on the system. +Such accesses shall instead be performed through the 3A API. The library +shall validate all accesses and restrict them to what is absolutely required +by 3A components. + +V4L2 Compatibility Layer +------------------------ + +The project shall support traditional V4L2 application through an additional +libcamera wrapper library. The wrapper library shall trap all accesses to +camera devices through `LD_PRELOAD`, and route them through libcamera to +emulate a high-level V4L2 camera device. It shall expose camera device +features on a best-effort basis, and aim for the level of features +traditionally available from a UVC camera designed for video conferencing. + +Android Camera HAL v3 Compatibility +----------------------------------- + +The library API shall expose all the features required to implement an +Android Camera HAL v3 on top of libcamera. Some features of the HAL may be +omitted as long as they can be implemented separately in the HAL, such as +JPEG encoding, or YUV reprocessing. diff --git a/Documentation/gen-doxyfile.py b/Documentation/gen-doxyfile.py new file mode 100755 index 00000000..c265bc2f --- /dev/null +++ b/Documentation/gen-doxyfile.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2024, Google Inc. +# +# Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> +# +# Generate Doxyfile from a template + +import argparse +import os +import string +import sys + + +def fill_template(template, data): + + template = open(template, 'rb').read() + template = template.decode('utf-8') + template = string.Template(template) + + return template.substitute(data) + + +def main(argv): + + parser = argparse.ArgumentParser() + parser.add_argument('-o', dest='output', metavar='file', + type=argparse.FileType('w', encoding='utf-8'), + default=sys.stdout, + help='Output file name (default: standard output)') + parser.add_argument('template', metavar='doxyfile.tmpl', type=str, + help='Doxyfile template') + parser.add_argument('inputs', type=str, nargs='*', + help='Input files') + + args = parser.parse_args(argv[1:]) + + inputs = [f'"{os.path.realpath(input)}"' for input in args.inputs] + data = fill_template(args.template, {'inputs': (' \\\n' + ' ' * 25).join(inputs)}) + args.output.write(data) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/Documentation/getting-started.rst b/Documentation/getting-started.rst index 987f43f7..63b050eb 100644 --- a/Documentation/getting-started.rst +++ b/Documentation/getting-started.rst @@ -1,4 +1,5 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 + .. Getting started information is defined in the project README file. .. include:: ../README.rst :start-after: .. section-begin-getting-started diff --git a/Documentation/guides/application-developer.rst b/Documentation/guides/application-developer.rst index c46d3362..6501345a 100644 --- a/Documentation/guides/application-developer.rst +++ b/Documentation/guides/application-developer.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: ../documentation-contents.rst + Using libcamera in a C++ application ==================================== @@ -116,19 +118,21 @@ available. .. code:: cpp - if (cm->cameras().empty()) { + auto cameras = cm->cameras(); + if (cameras.empty()) { std::cout << "No cameras were identified on the system." << std::endl; cm->stop(); return EXIT_FAILURE; } - std::string cameraId = cm->cameras()[0]->id(); - camera = cm->get(cameraId); + std::string cameraId = cameras[0]->id(); + camera = cm->get(cameraId); /* - * Note that is equivalent to: - * camera = cm->cameras()[0]; + * Note that `camera` may not compare equal to `cameras[0]`. + * In fact, it might simply be a `nullptr`, as the particular + * device might have disappeared (and reappeared) in the meantime. */ Once a camera has been selected an application needs to acquire an exclusive @@ -348,7 +352,7 @@ The libcamera library uses the concept of `signals and slots` (similar to `Qt Signals and Slots`_) to connect events with callbacks to handle them. .. _signals and slots: https://libcamera.org/api-html/classlibcamera_1_1Signal.html#details -.. _Qt Signals and Slots: https://doc.qt.io/qt-5/signalsandslots.html +.. _Qt Signals and Slots: https://doc.qt.io/qt-6/signalsandslots.html The ``Camera`` device emits two signals that applications can connect to in order to execute callbacks on frame completion events. @@ -479,7 +483,7 @@ instance. An example of how to write image data to disk is available in the `FileSink class`_ which is a part of the ``cam`` utility application in the libcamera repository. -.. _FileSink class: https://git.libcamera.org/libcamera/libcamera.git/tree/src/cam/file_sink.cpp +.. _FileSink class: https://git.libcamera.org/libcamera/libcamera.git/tree/src/apps/cam/file_sink.cpp With the handling of this request completed, it is possible to re-use the request and the associated buffers and re-queue it to the camera @@ -614,7 +618,7 @@ accordingly. In this example, the application file has been named simple_cam = executable('simple-cam', 'simple-cam.cpp', - dependencies: dependency('libcamera', required : true)) + dependencies: dependency('libcamera')) The ``dependencies`` line instructs meson to ask ``pkgconfig`` (or ``cmake``) to locate the ``libcamera`` library, which the test application will be diff --git a/Documentation/guides/introduction.rst b/Documentation/guides/introduction.rst deleted file mode 100644 index 700ec2d3..00000000 --- a/Documentation/guides/introduction.rst +++ /dev/null @@ -1,319 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-SA-4.0 - -Developers guide to libcamera -============================= - -The Linux kernel handles multimedia devices through the 'Linux media' subsystem -and provides a set of APIs (application programming interfaces) known -collectively as V4L2 (`Video for Linux 2`_) and the `Media Controller`_ API -which provide an interface to interact and control media devices. - -Included in this subsystem are drivers for camera sensors, CSI2 (Camera -Serial Interface) receivers, and ISPs (Image Signal Processors) - -The usage of these drivers to provide a functioning camera stack is a -responsibility that lies in userspace which is commonly implemented separately -by vendors without a common architecture or API for application developers. - -libcamera provides a complete camera stack for Linux based systems to abstract -functionality desired by camera application developers and process the -configuration of hardware and image control algorithms required to obtain -desirable results from the camera. - -.. _Video for Linux 2: https://www.linuxtv.org/downloads/v4l-dvb-apis-new/userspace-api/v4l/v4l2.html -.. _Media Controller: https://www.linuxtv.org/downloads/v4l-dvb-apis-new/userspace-api/mediactl/media-controller.html - - -In this developers guide, we will explore the `Camera Stack`_ and how it is -can be visualised at a high level, and explore the internal `Architecture`_ of -the libcamera library with its components. The current `Platform Support`_ is -detailed, as well as an overview of the `Licensing`_ requirements of the -project. - -This introduction is followed by a walkthrough tutorial to newcomers wishing to -support a new platform with the `Pipeline Handler Writers Guide`_ and for those -looking to make use of the libcamera native API an `Application Writers Guide`_ -provides a tutorial of the key APIs exposed by libcamera. - -.. _Pipeline Handler Writers Guide: pipeline-handler.html -.. _Application Writers Guide: application-developer.html - -.. TODO: Correctly link to the other articles of the guide - -Camera Stack ------------- - -The libcamera library is implemented in userspace, and makes use of underlying -kernel drivers that directly interact with hardware. - -Applications can make use of libcamera through the native `libcamera API`_'s or -through an adaptation layer integrating libcamera into a larger framework. - -.. _libcamera API: https://www.libcamera.org/api-html/index.html - -:: - - Application Layer - / +--------------+ +--------------+ +--------------+ +--------------+ - | | Native | | Framework | | Native | | Android | - | | V4L2 | | Application | | libcamera | | Camera | - | | Application | | (gstreamer) | | Application | | Framework | - \ +--------------+ +--------------+ +--------------+ +--------------+ - - ^ ^ ^ ^ - | | | | - | | | | - v v | v - Adaptation Layer | - / +--------------+ +--------------+ | +--------------+ - | | V4L2 | | gstreamer | | | Android | - | | Compatibility| | element | | | Camera | - | | (preload) | |(libcamerasrc)| | | HAL | - \ +--------------+ +--------------+ | +--------------+ - | - ^ ^ | ^ - | | | | - | | | | - v v v v - libcamera Framework - / +--------------------------------------------------------------------+ - | | | - | | libcamera | - | | | - \ +--------------------------------------------------------------------+ - - ^ ^ ^ - Userspace | | | - --------------------- | ---------------- | ---------------- | --------------- - Kernel | | | - v v v - - +-----------+ +-----------+ +-----------+ - | Media | <--> | Video | <--> | V4L2 | - | Device | | Device | | Subdev | - +-----------+ +-----------+ +-----------+ - -The camera stack comprises of four software layers. From bottom to top: - -* The kernel drivers control the camera hardware and expose a low-level - interface to userspace through the Linux kernel V4L2 family of APIs - (Media Controller API, V4L2 Video Device API and V4L2 Subdev API). - -* The libcamera framework is the core part of the stack. It handles all control - of the camera devices in its core component, libcamera, and exposes a native - C++ API to upper layers. - -* The libcamera adaptation layer is an umbrella term designating the components - that interface to libcamera in other frameworks. Notable examples are the V4L2 - compatibility layer, the gstreamer libcamera element, and the Android camera - HAL implementation based on libcamera which are provided as a part of the - libcamera project. - -* The applications and upper level frameworks are based on the libcamera - framework or libcamera adaptation, and are outside of the scope of the - libcamera project, however example native applications (cam, qcam) are - provided for testing. - - -V4L2 Compatibility Layer - V4L2 compatibility is achieved through a shared library that traps all - accesses to camera devices and routes them to libcamera to emulate high-level - V4L2 camera devices. It is injected in a process address space through - ``LD_PRELOAD`` and is completely transparent for applications. - - The compatibility layer exposes camera device features on a best-effort basis, - and aims for the level of features traditionally available from a UVC camera - designed for video conferencing. - -Android Camera HAL - Camera support for Android is achieved through a generic Android camera HAL - implementation on top of libcamera. The HAL implements features required by - Android and out of scope from libcamera, such as JPEG encoding support. - - This component is used to provide support for ChromeOS platforms - -GStreamer element (gstlibcamerasrc) - A `GStreamer element`_ is provided to allow capture from libcamera supported - devices through GStreamer pipelines, and connect to other elements for further - processing. - - Development of this element is ongoing and is limited to a single stream. - -Native libcamera API - Applications can make use of the libcamera API directly using the C++ - API. An example application and walkthrough using the libcamera API can be - followed in the `Application Writers Guide`_ - -.. _GStreamer element: https://gstreamer.freedesktop.org/documentation/application-development/basics/elements.html - -Architecture ------------- - -While offering a unified API towards upper layers, and presenting itself as a -single library, libcamera isn't monolithic. It exposes multiple components -through its public API and is built around a set of separate helpers internally. -Hardware abstractions are handled through the use of device-specific components -where required and dynamically loadable plugins are used to separate image -processing algorithms from the core libcamera codebase. - -:: - - --------------------------< libcamera Public API >--------------------------- - ^ ^ - | | - v v - +-------------+ +---------------------------------------------------+ - | Camera | | Camera Device | - | Manager | | +-----------------------------------------------+ | - +-------------+ | | Device-Agnostic | | - ^ | | | | - | | | +--------------------------+ | - | | | | ~~~~~~~~~~~~~~~~~~~~~~~ | - | | | | { +-----------------+ } | - | | | | } | //// Image //// | { | - | | | | <-> | / Processing // | } | - | | | | } | / Algorithms // | { | - | | | | { +-----------------+ } | - | | | | ~~~~~~~~~~~~~~~~~~~~~~~ | - | | | | ========================== | - | | | | +-----------------+ | - | | | | | // Pipeline /// | | - | | | | <-> | /// Handler /// | | - | | | | | /////////////// | | - | | +--------------------+ +-----------------+ | - | | Device-Specific | - | +---------------------------------------------------+ - | ^ ^ - | | | - v v v - +--------------------------------------------------------------------+ - | Helpers and Support Classes | - | +-------------+ +-------------+ +-------------+ +-------------+ | - | | MC & V4L2 | | Buffers | | Sandboxing | | Plugins | | - | | Support | | Allocator | | IPC | | Manager | | - | +-------------+ +-------------+ +-------------+ +-------------+ | - | +-------------+ +-------------+ | - | | Pipeline | | ... | | - | | Runner | | | | - | +-------------+ +-------------+ | - +--------------------------------------------------------------------+ - - /// Device-Specific Components - ~~~ Sandboxing - - -Camera Manager - The Camera Manager enumerates cameras and instantiates Pipeline Handlers to - manage each Camera that libcamera supports. The Camera Manager supports - hotplug detection and notification events when supported by the underlying - kernel devices. - - There is only ever one instance of the Camera Manager running per application. - Each application's instance of the Camera Manager ensures that only a single - application can take control of a camera device at once. - - Read the `Camera Manager API`_ documentation for more details. - -.. _Camera Manager API: https://libcamera.org/api-html/classlibcamera_1_1CameraManager.html - -Camera Device - The Camera class represents a single item of camera hardware that is capable - of producing one or more image streams, and provides the API to interact with - the underlying device. - - If a system has multiple instances of the same hardware attached, each has its - own instance of the camera class. - - The API exposes full control of the device to upper layers of libcamera through - the public API, making it the highest level object libcamera exposes, and the - object that all other API operations interact with from configuration to - capture. - - Read the `Camera API`_ documentation for more details. - -.. _Camera API: https://libcamera.org/api-html/classlibcamera_1_1Camera.html - -Pipeline Handler - The Pipeline Handler manages the complex pipelines exposed by the kernel - drivers through the Media Controller and V4L2 APIs. It abstracts pipeline - handling to hide device-specific details from the rest of the library, and - implements both pipeline configuration based on stream configuration, and - pipeline runtime execution and scheduling when needed by the device. - - The Pipeline Handler lives in the same process as the rest of the library, and - has access to all helpers and kernel camera-related devices. - - Hardware abstraction is handled by device specific Pipeline Handlers which are - derived from the Pipeline Handler base class allowing commonality to be shared - among the implementations. - - Derived pipeline handlers create Camera device instances based on the devices - they detect and support on the running system, and are responsible for - managing the interactions with a camera device. - - More details can be found in the `PipelineHandler API`_ documentation, and the - `Pipeline Handler Writers Guide`_. - -.. _PipelineHandler API: https://libcamera.org/api-html/classlibcamera_1_1PipelineHandler.html - -Image Processing Algorithms - An image processing algorithm (IPA) component is a loadable plugin that - implements 3A (Auto-Exposure, Auto-White Balance, and Auto-Focus) and other - algorithms. - - The algorithms run on the CPU and interact with the camera devices through the - Pipeline Handler to control hardware image processing based on the parameters - supplied by upper layers, maintaining state and closing the control loop - of the ISP. - - The component is sandboxed and can only interact with libcamera through the - API provided by the Pipeline Handler and an IPA has no direct access to kernel - camera devices. - - Open source IPA modules built with libcamera can be run in the same process - space as libcamera, however external IPA modules are run in a separate process - from the main libcamera process. IPA modules have a restricted view of the - system, including no access to networking APIs and limited access to file - systems. - - IPA modules are only required for platforms and devices with an ISP controlled - by the host CPU. Camera sensors which have an integrated ISP are not - controlled through the IPA module. - -Platform Support ----------------- - -The library currently supports the following hardware platforms specifically -with dedicated pipeline handlers: - - - Intel IPU3 (ipu3) - - Rockchip RK3399 (rkisp1) - - RaspberryPi 3 and 4 (rpi/vc4) - -Furthermore, generic platform support is provided for the following: - - - USB video device class cameras (uvcvideo) - - iMX7, Allwinner Sun6i (simple) - - Virtual media controller driver for test use cases (vimc) - -Licensing ---------- - -The libcamera core, is covered by the `LGPL-2.1-or-later`_ license. Pipeline -Handlers are a part of the libcamera code base and need to be contributed -upstream by device vendors. IPA modules included in libcamera are covered by a -free software license, however third-parties may develop IPA modules outside of -libcamera and distribute them under a closed-source license, provided they do -not include source code from the libcamera project. - -The libcamera project itself contains multiple libraries, applications and -utilities. Licenses are expressed through SPDX tags in text-based files that -support comments, and through the .reuse/dep5 file otherwise. A copy of all -licenses are stored in the LICENSES directory, and a full summary of the -licensing used throughout the project can be found in the COPYING.rst document. - -Applications which link dynamically against libcamera and use only the public -API are an independent work of the authors and have no license restrictions -imposed upon them from libcamera. - -.. _LGPL-2.1-or-later: https://spdx.org/licenses/LGPL-2.1-or-later.html diff --git a/Documentation/guides/ipa.rst b/Documentation/guides/ipa.rst index 25deadef..cd640563 100644 --- a/Documentation/guides/ipa.rst +++ b/Documentation/guides/ipa.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: ../documentation-contents.rst + IPA Writer's Guide ================== diff --git a/Documentation/guides/pipeline-handler.rst b/Documentation/guides/pipeline-handler.rst index 10b9c75c..fe752975 100644 --- a/Documentation/guides/pipeline-handler.rst +++ b/Documentation/guides/pipeline-handler.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: ../documentation-contents.rst + Pipeline Handler Writers Guide ============================== @@ -151,13 +153,14 @@ integrates with the libcamera build system, and a *vivid.cpp* file that matches the name of the pipeline. In the *meson.build* file, add the *vivid.cpp* file as a build source for -libcamera by adding it to the global meson ``libcamera_sources`` variable: +libcamera by adding it to the global meson ``libcamera_internal_sources`` +variable: .. code-block:: none # SPDX-License-Identifier: CC0-1.0 - libcamera_sources += files([ + libcamera_internal_sources += files([ 'vivid.cpp', ]) @@ -183,7 +186,7 @@ to the libcamera build options in the top level ``meson_options.txt``. option('pipelines', type : 'array', - choices : ['ipu3', 'rkisp1', 'rpi/vc4', 'simple', 'uvcvideo', 'vimc', 'vivid'], + choices : ['ipu3', 'rkisp1', 'rpi/pisp', 'rpi/vc4', 'simple', 'uvcvideo', 'vimc', 'vivid'], description : 'Select which pipeline handlers to include') @@ -258,7 +261,7 @@ implementations for the overridden class members. return false; } - REGISTER_PIPELINE_HANDLER(PipelineHandlerVivid) + REGISTER_PIPELINE_HANDLER(PipelineHandlerVivid, "vivid") } /* namespace libcamera */ @@ -266,6 +269,8 @@ Note that you must register the ``PipelineHandler`` subclass with the pipeline handler factory using the `REGISTER_PIPELINE_HANDLER`_ macro which registers it and creates a global symbol to reference the class and make it available to try and match devices. +String "vivid" is the name assigned to the pipeline, matching the pipeline +subdirectory name in the source tree. .. _REGISTER_PIPELINE_HANDLER: https://libcamera.org/api-html/pipeline__handler_8h.html @@ -516,14 +521,14 @@ handler and camera manager using `registerCamera`_. Finally with a successful construction, we return 'true' indicating that the PipelineHandler successfully matched and constructed a device. -.. _Camera::create: https://libcamera.org/api-html/classlibcamera_1_1Camera.html#a453740e0d2a2f495048ae307a85a2574 +.. _Camera::create: https://libcamera.org/internal-api-html/classlibcamera_1_1Camera.html#adf5e6c22411f953bfaa1ae21155d6c31 .. _registerCamera: https://libcamera.org/api-html/classlibcamera_1_1PipelineHandler.html#adf02a7f1bbd87aca73c0e8d8e0e6c98b .. code-block:: cpp std::set<Stream *> streams{ &data->stream_ }; - std::shared_ptr<Camera> camera = Camera::create(this, data->video_->deviceName(), streams); - registerCamera(std::move(camera), std::move(data)); + std::shared_ptr<Camera> camera = Camera::create(std::move(data), data->video_->deviceName(), streams); + registerCamera(std::move(camera)); return true; @@ -549,8 +554,7 @@ Our match function should now look like the following: /* Create and register the camera. */ std::set<Stream *> streams{ &data->stream_ }; - const std::string &id = data->video_->deviceName(); - std::shared_ptr<Camera> camera = Camera::create(data.release(), id, streams); + std::shared_ptr<Camera> camera = Camera::create(std::move(data), data->video_->deviceName(), streams); registerCamera(std::move(camera)); return true; @@ -587,12 +591,12 @@ immutable properties of the ``Camera`` device. The libcamera controls and properties are defined in YAML form which is processed to automatically generate documentation and interfaces. Controls are -defined by the src/libcamera/`control_ids.yaml`_ file and camera properties -are defined by src/libcamera/`properties_ids.yaml`_. +defined by the src/libcamera/`control_ids_core.yaml`_ file and camera properties +are defined by src/libcamera/`property_ids_core.yaml`_. .. _controls framework: https://libcamera.org/api-html/controls_8h.html -.. _control_ids.yaml: https://libcamera.org/api-html/control__ids_8h.html -.. _properties_ids.yaml: https://libcamera.org/api-html/property__ids_8h.html +.. _control_ids_core.yaml: https://libcamera.org/api-html/control__ids_8h.html +.. _property_ids_core.yaml: https://libcamera.org/api-html/property__ids_8h.html Pipeline handlers can optionally register the list of controls an application can set as well as a list of immutable camera properties. Being both @@ -651,7 +655,7 @@ inline in our VividCameraData init: ctrls.emplace(id, info); } - controlInfo_ = std::move(ctrls); + controlInfo_ = ControlInfoMap(std::move(ctrls), controls::controls); The ``properties_`` field is a list of ``ControlId`` instances associated with immutable values, which represent static characteristics that can @@ -672,6 +676,58 @@ handling controls: #include <libcamera/controls.h> #include <libcamera/control_ids.h> +Vendor-specific controls and properties +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Vendor-specific controls and properties must be defined in a separate YAML file +and included in the build by defining the pipeline handler to file mapping in +include/libcamera/meson.build. These YAML files live in the src/libcamera +directory. + +For example, adding a Raspberry Pi vendor control file for the PiSP pipeline +handler is done with the following mapping: + +.. code-block:: meson + + controls_map = { + 'controls': { + 'draft': 'control_ids_draft.yaml', + 'libcamera': 'control_ids_core.yaml', + 'rpi/pisp': 'control_ids_rpi.yaml', + }, + + 'properties': { + 'draft': 'property_ids_draft.yaml', + 'libcamera': 'property_ids_core.yaml', + } + } + +The pipeline handler named above must match the pipeline handler option string +specified in the meson build configuration. + +Vendor-specific controls and properties must contain a `vendor: <vendor_string>` +tag in the YAML file. Every unique vendor tag must define a unique and +non-overlapping range of reserved control IDs in src/libcamera/control_ranges.yaml. + +For example, the following block defines a vendor-specific control with the +`rpi` vendor tag: + +.. code-block:: yaml + + vendor: rpi + controls: + - PispConfigDumpFile: + type: string + description: | + Triggers the Raspberry Pi PiSP pipeline handler to generate a JSON + formatted dump of the Backend configuration to the filename given by the + value of the control. + +The controls will be generated in the vendor-specific namespace +`libcamera::controls::rpi`. Additionally a `#define +LIBCAMERA_HAS_RPI_VENDOR_CONTROLS` will be available to allow applications to +test for the availability of these controls. + Generating a default configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -743,8 +799,7 @@ derived class, and assign it to a base class pointer. .. code-block:: cpp - VividCameraData *data = cameraData(camera); - CameraConfiguration *config = new VividCameraConfiguration(); + auto config = std::make_unique<VividCameraConfiguration>(); A ``CameraConfiguration`` is specific to each pipeline, so you can only create it from the pipeline handler code path. Applications can also generate an empty @@ -772,9 +827,7 @@ To generate a ``StreamConfiguration``, you need a list of pixel formats and frame sizes which are supported as outputs of the stream. You can fetch a map of the ``V4LPixelFormat`` and ``SizeRange`` supported by the underlying output device, but the pipeline handler needs to convert this to a -``libcamera::PixelFormat`` type to pass to applications. We do this here using -``std::transform`` to convert the formats and populate a new ``PixelFormat`` map -as shown below. +``libcamera::PixelFormat`` type to pass to applications. Continue adding the following code example to our ``generateConfiguration`` implementation. @@ -784,14 +837,12 @@ implementation. std::map<V4L2PixelFormat, std::vector<SizeRange>> v4l2Formats = data->video_->formats(); std::map<PixelFormat, std::vector<SizeRange>> deviceFormats; - std::transform(v4l2Formats.begin(), v4l2Formats.end(), - std::inserter(deviceFormats, deviceFormats.begin()), - [&](const decltype(v4l2Formats)::value_type &format) { - return decltype(deviceFormats)::value_type{ - format.first.toPixelFormat(), - format.second - }; - }); + + for (auto &[v4l2PixelFormat, sizes] : v4l2Formats) { + PixelFormat pixelFormat = v4l2PixelFormat.toPixelFormat(); + if (pixelFormat.isValid()) + deviceFormats.try_emplace(pixelFormat, std::move(sizes)); + } The `StreamFormats`_ class holds information about the pixel formats and frame sizes that a stream can support. The class groups size information by the pixel @@ -881,9 +932,9 @@ Add the following function implementation to your file: StreamConfiguration &cfg = config_[0]; - const std::vector<libcamera::PixelFormat> formats = cfg.formats().pixelformats(); + const std::vector<libcamera::PixelFormat> &formats = cfg.formats().pixelformats(); if (std::find(formats.begin(), formats.end(), cfg.pixelFormat) == formats.end()) { - cfg.pixelFormat = cfg.formats().pixelformats()[0]; + cfg.pixelFormat = formats[0]; LOG(VIVID, Debug) << "Adjusting format to " << cfg.pixelFormat.toString(); status = Adjusted; } @@ -1293,7 +1344,7 @@ before being set. continue; } - int32_t value = lroundf(it.second.get<float>() * 128 + offset); + int32_t value = std::lround(it.second.get<float>() * 128 + offset); controls.set(cid, std::clamp(value, 0, 255)); } @@ -1357,7 +1408,7 @@ value translation operations: .. code-block:: cpp - #include <math.h> + #include <cmath> Frame completion and event handling ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1370,7 +1421,7 @@ emitted triggers the execution of the connected slots. A detailed description of the libcamera implementation is available in the `libcamera Signal and Slot`_ classes documentation. -.. _Qt Signals and Slots: https://doc.qt.io/qt-5/signalsandslots.html +.. _Qt Signals and Slots: https://doc.qt.io/qt-6/signalsandslots.html .. _libcamera Signal and Slot: https://libcamera.org/api-html/classlibcamera_1_1Signal.html#details In order to notify applications about the availability of new frames and data, diff --git a/Documentation/guides/tracing.rst b/Documentation/guides/tracing.rst index ae960d85..537dce50 100644 --- a/Documentation/guides/tracing.rst +++ b/Documentation/guides/tracing.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: ../documentation-contents.rst + Tracing Guide ============= diff --git a/Documentation/index.rst b/Documentation/index.rst index 63fac72d..251112fb 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -1,26 +1,31 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 -.. Front page matter is defined in the project README file. -.. include:: ../README.rst - :start-after: .. section-begin-libcamera - :end-before: .. section-end-libcamera +.. include:: introduction.rst .. toctree:: :maxdepth: 1 :caption: Contents: Home <self> - Docs <docs> Contribute <contributing> Getting Started <getting-started> - Developer Guide <guides/introduction> Application Writer's Guide <guides/application-developer> - Pipeline Handler Writer's Guide <guides/pipeline-handler> - IPA Writer's guide <guides/ipa> - Tracing guide <guides/tracing> + Camera Sensor Model <camera-sensor-model> Environment variables <environment_variables> - Sensor driver requirements <sensor_driver_requirements> + Feature Requirements <feature_requirements> + IPA Writer's guide <guides/ipa> Lens driver requirements <lens_driver_requirements> + libcamera Architecture <libcamera_architecture> + Pipeline Handler Writer's Guide <guides/pipeline-handler> Python Bindings <python-bindings> - Camera Sensor Model <camera-sensor-model> + Sensor driver requirements <sensor_driver_requirements> + SoftwareISP Benchmarking <software-isp-benchmarking> + Tracing guide <guides/tracing> + + Design document: AE <design/ae> + +.. toctree:: + :hidden: + + introduction diff --git a/Documentation/internal-api-html/index.rst b/Documentation/internal-api-html/index.rst new file mode 100644 index 00000000..43768648 --- /dev/null +++ b/Documentation/internal-api-html/index.rst @@ -0,0 +1,8 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +.. _internal-api: + +Internal API Reference +====================== + +:: Placeholder for Doxygen documentation diff --git a/Documentation/introduction.rst b/Documentation/introduction.rst new file mode 100644 index 00000000..82aa11a3 --- /dev/null +++ b/Documentation/introduction.rst @@ -0,0 +1,224 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +.. include:: documentation-contents.rst + +************ +Introduction +************ + +.. toctree:: + :hidden: + + API <api-html/index> + Internal API <internal-api-html/index> + +What is libcamera? +================== + +libcamera is an open source complex camera support library for Linux, Android +and ChromeOS. The library interfaces with Linux kernel device drivers and +provides an intuitive API to developers in order to simplify the complexity +involved in capturing images from complex cameras on Linux systems. + +What is a "complex camera"? +=========================== + +A modern "camera" tends to infact be several different pieces of hardware which +must all be controlled together in order to produce and capture images of +appropriate quality. A hardware pipeline typically consists of a camera sensor +that captures raw frames and transmits them on a bus, a receiver that decodes +the bus signals, and an image signal processor that processes raw frames to +produce usable images in a standard format. The Linux kernel handles these +multimedia devices through the 'Linux media' subsystem and provides a set of +application programming interfaces known collectively as the +V4L2 (`Video for Linux 2`_) and the `Media Controller`_ APIs, which provide an +interface to interact and control media devices. + +.. _Video for Linux 2: https://www.linuxtv.org/downloads/v4l-dvb-apis-new/userspace-api/v4l/v4l2.html +.. _Media Controller: https://www.linuxtv.org/downloads/v4l-dvb-apis-new/userspace-api/mediactl/media-controller.html + +Included in this subsystem are drivers for camera sensors, CSI2 (Camera +Serial Interface) receivers, and ISPs (Image Signal Processors). + +The usage of these drivers to provide a functioning camera stack is a +responsibility that lies in userspace, and is commonly implemented separately +by vendors without a common architecture or API for application developers. This +adds a lot of complexity to the task, particularly when considering that the +differences in hardware pipelines and their representation in the kernel's APIs +often necessitate bespoke handling. + +What is libcamera for? +====================== + +libcamera provides a complete camera stack for Linux-based systems to abstract +the configuration of hardware and image control algorithms required to obtain +desirable results from the camera through the kernel's APIs, reducing those +operations to a simple and consistent method for developers. In short instead of +having to deal with this: + +.. graphviz:: mali-c55.dot + +you can instead simply deal with: + +.. code-block:: python + + >>> import libcamera as lc + >>> camera_manager = lc.CameraManager.singleton() + [0:15:59.582029920] [504] INFO Camera camera_manager.cpp:313 libcamera v0.3.0+182-01e57380 + >>> for camera in camera_manager.cameras: + ... print(f' - {camera.id}') + ... + - mali-c55 tpg + - imx415 1-001a + +The library handles the rest for you. These documentary pages give more +information on the internal workings of libcamera (and the kernel camera stack +that lies behind it) as well as guidance on using libcamera in an application or +extending the library with support for your hardware (through the pipeline +handler and IPA module writer's guides). + +How should I use it? +==================== + +There are a few ways you might want to use libcamera, depending on your +application. It's always possible to use the library directly, and you can find +detailed information on how to do so in the +:doc:`application writer's guide <guides/application-developer>`. + +It is often more appropriate to use one of the frameworks with libcamera +support. For example an application powering an embedded media device +incorporating capture, encoding and streaming of both video and audio would +benefit from using `GStreamer`_, for which libcamera provides a plugin. +Similarly an application for user-facing devices like a laptop would likely +benefit accessing cameras through the XDG camera portal and `pipewire`_, which +brings the advantages of resource sharing (multiple applications accessing the +stream at the same time) and access control. + +.. _GStreamer: https://gstreamer.freedesktop.org/ +.. _pipewire: https://pipewire.org/ + +Camera Stack +============ + +:: + + a c / +-------------+ +-------------+ +-------------+ +-------------+ + p a | | Native | | Framework | | Native | | Android | + p t | | V4L2 | | Application | | libcamera | | Camera | + l i | | Application | | (gstreamer) | | Application | | Framework | + i o \ +-------------+ +-------------+ +-------------+ +-------------+ + n ^ ^ ^ ^ + | | | | + l a | | | | + i d v v | v + b a / +-------------+ +-------------+ | +-------------+ + c p | | V4L2 | | Camera | | | Android | + a t | | Compat. | | Framework | | | Camera | + m a | | | | (gstreamer) | | | HAL | + e t \ +-------------+ +-------------+ | +-------------+ + r i ^ ^ | ^ + a o | | | | + n | | | | + / | ,................................................ + | | ! : Language : ! + l f | | ! : Bindings : ! + i r | | ! : (optional) : ! + b a | | \...............................................' + c m | | | | | + a e | | | | | + m w | v v v v + e o | +----------------------------------------------------------------+ + r r | | | + a k | | libcamera | + | | | + \ +----------------------------------------------------------------+ + ^ ^ ^ + Userspace | | | + ------------------------ | ---------------- | ---------------- | --------------- + Kernel | | | + v v v + +-----------+ +-----------+ +-----------+ + | Media | <--> | Video | <--> | V4L2 | + | Device | | Device | | Subdev | + +-----------+ +-----------+ +-----------+ + +The camera stack comprises four software layers. From bottom to top: + +* The kernel drivers control the camera hardware and expose a + low-level interface to userspace through the Linux kernel V4L2 + family of APIs (Media Controller API, V4L2 Video Device API and + V4L2 Subdev API). + +* The libcamera framework is the core part of the stack. It + handles all control of the camera devices in its core component, + libcamera, and exposes a native C++ API to upper layers. Optional + language bindings allow interfacing to libcamera from other + programming languages. + + Those components live in the same source code repository and + all together constitute the libcamera framework. + +* The libcamera adaptation is an umbrella term designating the + components that interface to libcamera in other frameworks. + Notable examples are a V4L2 compatibility layer, a gstreamer + libcamera element, and an Android camera HAL implementation based + on libcamera. + + Those components can live in the libcamera project source code + in separate repositories, or move to their respective project's + repository (for instance the gstreamer libcamera element). + +* The applications and upper level frameworks are based on the + libcamera framework or libcamera adaptation, and are outside of + the scope of the libcamera project. + +V4L2 Compatibility Layer + V4L2 compatibility is achieved through a shared library that traps all + accesses to camera devices and routes them to libcamera to emulate high-level + V4L2 camera devices. It is injected in a process address space through + ``LD_PRELOAD`` and is completely transparent for applications. + + The compatibility layer exposes camera device features on a best-effort basis, + and aims for the level of features traditionally available from a UVC camera + designed for video conferencing. + +Android Camera HAL + Camera support for Android is achieved through a generic Android camera HAL + implementation on top of libcamera. The HAL implements features required by + Android and out of scope from libcamera, such as JPEG encoding support. + + This component is used to provide support for ChromeOS platforms. + +GStreamer element (gstlibcamerasrc) + A `GStreamer element`_ is provided to allow capture from libcamera supported + devices through GStreamer pipelines, and connect to other elements for further + processing. + +Native libcamera API + Applications can make use of the libcamera API directly using the C++ + API. An example application and walkthrough using the libcamera API can be + followed in the :doc:`Application writer's guide </guides/application-developer>` + +.. _GStreamer element: https://gstreamer.freedesktop.org/documentation/application-development/basics/elements.html + +Licensing +========= + +The libcamera core is covered by the `LGPL-2.1-or-later`_ license. Pipeline +Handlers are a part of the libcamera code base and need to be contributed +upstream by device vendors. IPA modules included in libcamera are covered by a +free software license, however third-parties may develop IPA modules outside of +libcamera and distribute them under a closed-source license, provided they do +not include source code from the libcamera project. + +The libcamera project itself contains multiple libraries, applications and +utilities. Licenses are expressed through SPDX tags in text-based files that +support comments, and through the .reuse/dep5 file otherwise. A copy of all +licenses are stored in the LICENSES directory, and a full summary of the +licensing used throughout the project can be found in the COPYING.rst document. + +Applications which link dynamically against libcamera and use only the public +API are an independent work of the authors and have no license restrictions +imposed upon them from libcamera. + +.. _LGPL-2.1-or-later: https://spdx.org/licenses/LGPL-2.1-or-later.html diff --git a/Documentation/lens_driver_requirements.rst b/Documentation/lens_driver_requirements.rst index b96e502d..85fef76f 100644 --- a/Documentation/lens_driver_requirements.rst +++ b/Documentation/lens_driver_requirements.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: documentation-contents.rst + .. _lens-driver-requirements: Lens Driver Requirements diff --git a/Documentation/libcamera_architecture.rst b/Documentation/libcamera_architecture.rst new file mode 100644 index 00000000..abbb0d17 --- /dev/null +++ b/Documentation/libcamera_architecture.rst @@ -0,0 +1,168 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +.. include:: documentation-contents.rst + +libcamera Architecture +====================== + +While offering a unified API towards upper layers, and presenting itself as a +single library, libcamera isn't monolithic. It exposes multiple components +through its public API and is built around a set of separate helpers internally. +Hardware abstractions are handled through the use of device-specific components +where required and dynamically loadable plugins are used to separate image +processing algorithms from the core libcamera codebase. + +:: + + --------------------------< libcamera Public API >--------------------------- + ^ ^ + | | + v v + +-------------+ +---------------------------------------------------+ + | Camera | | Camera Device | + | Manager | | +-----------------------------------------------+ | + +-------------+ | | Device-Agnostic | | + ^ | | | | + | | | +--------------------------+ | + | | | | ~~~~~~~~~~~~~~~~~~~~~~~ | + | | | | { +-----------------+ } | + | | | | } | //// Image //// | { | + | | | | <-> | / Processing // | } | + | | | | } | / Algorithms // | { | + | | | | { +-----------------+ } | + | | | | ~~~~~~~~~~~~~~~~~~~~~~~ | + | | | | ========================== | + | | | | +-----------------+ | + | | | | | // Pipeline /// | | + | | | | <-> | /// Handler /// | | + | | | | | /////////////// | | + | | +--------------------+ +-----------------+ | + | | Device-Specific | + | +---------------------------------------------------+ + | ^ ^ + | | | + v v v + +--------------------------------------------------------------------+ + | Helpers and Support Classes | + | +-------------+ +-------------+ +-------------+ +-------------+ | + | | MC & V4L2 | | Buffers | | Sandboxing | | Plugins | | + | | Support | | Allocator | | IPC | | Manager | | + | +-------------+ +-------------+ +-------------+ +-------------+ | + | +-------------+ +-------------+ | + | | Pipeline | | ... | | + | | Runner | | | | + | +-------------+ +-------------+ | + +--------------------------------------------------------------------+ + + /// Device-Specific Components + ~~~ Sandboxing + + +Camera Manager + The Camera Manager enumerates cameras and instantiates Pipeline Handlers to + manage each Camera that libcamera supports. The Camera Manager supports + hotplug detection and notification events when supported by the underlying + kernel devices. + + There is only ever one instance of the Camera Manager running per application. + Each application's instance of the Camera Manager ensures that only a single + application can take control of a camera device at once. + + Read the `Camera Manager API`_ documentation for more details. + +.. _Camera Manager API: https://libcamera.org/api-html/classlibcamera_1_1CameraManager.html + +Camera Device + The Camera class represents a single item of camera hardware that is capable + of producing one or more image streams, and provides the API to interact with + the underlying device. + + If a system has multiple instances of the same hardware attached, each has its + own instance of the camera class. + + The API exposes full control of the device to upper layers of libcamera through + the public API, making it the highest level object libcamera exposes, and the + object that all other API operations interact with from configuration to + capture. + + Read the `Camera API`_ documentation for more details. + +.. _Camera API: https://libcamera.org/api-html/classlibcamera_1_1Camera.html + +Pipeline Handler + The Pipeline Handler manages the complex pipelines exposed by the kernel + drivers through the Media Controller and V4L2 APIs. It abstracts pipeline + handling to hide device-specific details from the rest of the library, and + implements both pipeline configuration based on stream configuration, and + pipeline runtime execution and scheduling when needed by the device. + + The Pipeline Handler lives in the same process as the rest of the library, and + has access to all helpers and kernel camera-related devices. + + Hardware abstraction is handled by device specific Pipeline Handlers which are + derived from the Pipeline Handler base class allowing commonality to be shared + among the implementations. + + Derived pipeline handlers create Camera device instances based on the devices + they detect and support on the running system, and are responsible for + managing the interactions with a camera device. + + More details can be found in the `PipelineHandler API`_ documentation, and the + :doc:`Pipeline Handler Writers Guide <guides/pipeline-handler>`. + +.. _PipelineHandler API: https://libcamera.org/api-html/classlibcamera_1_1PipelineHandler.html + +Image Processing Algorithms + Together with the hardware image processing and hardware statistics + collection, the Image Processing Algorithms (IPA) implement 3A (Auto-Exposure, + Auto-White Balance and Auto-Focus) and other algorithms. They run on the CPU + and control hardware image processing based on the parameters supplied by + upper layers, closing the control loop of the ISP. + + IPAs are loaded as external plugins named IPA Modules. IPA Modules can be part + of the libcamera code base or provided externally by camera vendors as + open-source or closed-source components. + + Open source IPA Modules built with libcamera are run in the same process space + as libcamera. External IPA Modules are run in a separate sandboxed process. In + either case, they can only interact with libcamera through the API provided by + the Pipeline Handler. They have a restricted view of the system, with no direct + access to kernel camera devices, no access to networking APIs, and limited + access to file systems. All their accesses to image and metadata are mediated + by dmabuf instances explicitly passed by the Pipeline Handler to the IPA + Module. + + IPA Modules are only required for platforms and devices with an ISP controlled + by the host CPU. Camera sensors which have an integrated ISP are not + controlled through the IPA Module. + +Helpers and Support Classes + While Pipeline Handlers are device-specific, implementations are expected to + share code due to usage of identical APIs towards the kernel camera drivers + and the Image Processing Algorithms. This includes without limitation handling + of the MC and V4L2 APIs, buffer management through dmabuf, and pipeline + discovery, configuration and scheduling. Such code will be factored out to + helpers when applicable. + + Other parts of libcamera will also benefit from factoring code out to + self-contained support classes, even if such code is present only once in the + code base, in order to keep the source code clean and easy to read. This + should be the case for instance for plugin management. + +Platform Support +---------------- + +The library currently supports the following hardware platforms specifically +with dedicated pipeline handlers: + + - Arm Mali-C55 + - Intel IPU3 (ipu3) + - NXP i.MX8MP (imx8-isi and rkisp1) + - RaspberryPi 3, 4 and zero (rpi/vc4) + - Rockchip RK3399 (rkisp1) + +Furthermore, generic platform support is provided for the following: + + - USB video device class cameras (uvcvideo) + - iMX7, IPU6, Allwinner Sun6i (simple) + - Virtual media controller driver for test use cases (vimc) diff --git a/Documentation/mainpage.dox b/Documentation/mainpage.dox new file mode 100644 index 00000000..cbee9bab --- /dev/null +++ b/Documentation/mainpage.dox @@ -0,0 +1,33 @@ +/** +\mainpage libcamera API reference + +Welcome to the API reference for <a href="https://libcamera.org/">libcamera</a>, +a complex camera support library for Linux, Android and ChromeOS. These pages +are automatically generated from the libcamera source code and describe the API +in detail - if this is your first interaction with libcamera then you may find +it useful to visit the [documentation](../introduction.html) in +the first instance, which can provide a more generic introduction to the +library's concepts. + +\if internal + +As a follow-on to the developer's guide, to assist you in adding support for +your platform the [pipeline handler writer's guide](../guides/pipeline-handler.html) +and the [ipa module writer's guide](../guides/ipa.html) should be helpful. + +The full libcamera API is documented here. If you wish to see only the public +part of the API you can use [these pages](../api-html/index.html) instead. + +\else + +As a follow-on to the developer's guide, to assist you in using libcamera within +your project the [application developer's guide](../guides/application-developer.html) +gives an overview on how to achieve that. + +Only the public part of the libcamera API is documented here; if you are a +developer seeking to add support for your hardware to the library or make other +improvements, you should switch to the internal API +[reference pages](../internal-api-html/index.html) instead. + +\endif +*/ diff --git a/Documentation/mali-c55.dot b/Documentation/mali-c55.dot new file mode 100644 index 00000000..7bfc44c0 --- /dev/null +++ b/Documentation/mali-c55.dot @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: CC-BY-SA-4.0 */ + +digraph board { + rankdir=TB + n00000001 [label="{{} | mali-c55 tpg\n/dev/v4l-subdev0 | {<port0> 0}}", shape=Mrecord, style=filled, fillcolor=green] + n00000001:port0 -> n00000003:port0 [style=dashed] + n00000003 [label="{{<port0> 0 | <port4> 4} | mali-c55 isp\n/dev/v4l-subdev1 | {<port1> 1 | <port2> 2 | <port3> 3}}", shape=Mrecord, style=filled, fillcolor=green] + n00000003:port1 -> n00000009:port0 [style=bold] + n00000003:port2 -> n00000009:port2 [style=bold] + n00000003:port1 -> n0000000d:port0 [style=bold] + n00000003:port3 -> n0000001c + n00000009 [label="{{<port0> 0 | <port2> 2} | mali-c55 resizer fr\n/dev/v4l-subdev2 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n00000009:port1 -> n00000010 + n0000000d [label="{{<port0> 0} | mali-c55 resizer ds\n/dev/v4l-subdev3 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n0000000d:port1 -> n00000014 + n00000010 [label="mali-c55 fr\n/dev/video0", shape=box, style=filled, fillcolor=yellow] + n00000014 [label="mali-c55 ds\n/dev/video1", shape=box, style=filled, fillcolor=yellow] + n00000018 [label="mali-c55 3a params\n/dev/video2", shape=box, style=filled, fillcolor=yellow] + n00000018 -> n00000003:port4 + n0000001c [label="mali-c55 3a stats\n/dev/video3", shape=box, style=filled, fillcolor=yellow] + n00000030 [label="{{<port0> 0} | lte-csi2-rx\n/dev/v4l-subdev4 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n00000030:port1 -> n00000003:port0 + n00000035 [label="{{} | imx415 1-001a\n/dev/v4l-subdev5 | {<port0> 0}}", shape=Mrecord, style=filled, fillcolor=green] + n00000035:port0 -> n00000030:port0 [style=bold] +} diff --git a/Documentation/meson.build b/Documentation/meson.build index 7c150259..0fc5909d 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -15,6 +15,7 @@ if doxygen.found() and dot.found() cdata.set('TOP_SRCDIR', meson.project_source_root()) cdata.set('TOP_BUILDDIR', meson.project_build_root()) cdata.set('OUTPUT_DIR', meson.current_build_dir()) + cdata.set('WARN_AS_ERROR', get_option('doc_werror') ? 'YES' : 'NO') doxygen_predefined = [] foreach key : config_h.keys() @@ -23,61 +24,124 @@ if doxygen.found() and dot.found() cdata.set('PREDEFINED', ' \\\n\t\t\t '.join(doxygen_predefined)) - doxyfile = configure_file(input : 'Doxyfile.in', - output : 'Doxyfile', - configuration : cdata) + doxyfile_common = configure_file(input : 'Doxyfile-common.in', + output : 'Doxyfile-common', + configuration : cdata) + + doxygen_public_input = [ + libcamera_base_public_headers, + libcamera_base_public_sources, + libcamera_public_headers, + libcamera_public_sources, + ] - doxygen_input = [ - doxyfile, - libcamera_base_headers, - libcamera_base_sources, + doxygen_internal_input = [ + libcamera_base_private_headers, + libcamera_base_internal_sources, libcamera_internal_headers, + libcamera_internal_sources, libcamera_ipa_headers, libcamera_ipa_interfaces, - libcamera_public_headers, - libcamera_sources, libipa_headers, libipa_sources, ] if is_variable('ipu3_ipa_sources') - doxygen_input += [ipu3_ipa_sources] + doxygen_internal_input += [ipu3_ipa_sources] endif - custom_target('doxygen', - input : doxygen_input, + # We run doxygen twice - the first run excludes internal API objects as it + # is intended to document the public API only. A second run covers all of + # the library's objects for libcamera developers. Common configuration is + # set in an initially generated Doxyfile, which is then included by the two + # final Doxyfiles. + + # This is the "public" run of doxygen generating an abridged version of the + # API's documentation. + + doxyfile_tmpl = configure_file(input : 'Doxyfile-public.in', + output : 'Doxyfile-public.tmpl', + configuration : cdata) + + # The set of public input files stored in the doxygen_public_input array + # needs to be set in Doxyfile public. We can't pass them through cdata + # cdata, as some of the array members are custom_tgt instances, which + # configuration_data.set() doesn't support. Using a separate script invoked + # through custom_target(), which supports custom_tgt instances as inputs. + + doxyfile = custom_target('doxyfile-public', + input : [ + doxygen_public_input, + ], + output : 'Doxyfile-public', + command : [ + 'gen-doxyfile.py', + '-o', '@OUTPUT@', + doxyfile_tmpl, + '@INPUT@', + ]) + + custom_target('doxygen-public', + input : [ + doxyfile, + doxyfile_common, + ], output : 'api-html', command : [doxygen, doxyfile], install : true, - install_dir : doc_install_dir) + install_dir : doc_install_dir, + install_tag : 'doc') + + # This is the internal documentation, which hard-codes a list of directories + # to parse in its doxyfile. + + doxyfile = configure_file(input : 'Doxyfile-internal.in', + output : 'Doxyfile-internal', + configuration : cdata) + + custom_target('doxygen-internal', + input : [ + doxyfile, + doxyfile_common, + doxygen_internal_input, + ], + output : 'internal-api-html', + command : [doxygen, doxyfile], + install : true, + install_dir : doc_install_dir, + install_tag : 'doc-internal') endif # # Sphinx # -sphinx = find_program('sphinx-build-3', required : false) -if not sphinx.found() - sphinx = find_program('sphinx-build', required : get_option('documentation')) -endif +sphinx = find_program('sphinx-build-3', 'sphinx-build', + required : get_option('documentation')) if sphinx.found() docs_sources = [ 'camera-sensor-model.rst', + 'code-of-conduct.rst', 'coding-style.rst', 'conf.py', 'contributing.rst', - 'docs.rst', + 'design/ae.rst', + 'documentation-contents.rst', 'environment_variables.rst', + 'feature_requirements.rst', 'guides/application-developer.rst', - 'guides/introduction.rst', 'guides/ipa.rst', 'guides/pipeline-handler.rst', 'guides/tracing.rst', 'index.rst', + 'introduction.rst', 'lens_driver_requirements.rst', + 'libcamera_architecture.rst', + 'mali-c55.dot', 'python-bindings.rst', 'sensor_driver_requirements.rst', + 'software-isp-benchmarking.rst', '../README.rst', ] @@ -90,7 +154,8 @@ if sphinx.found() output : 'html', build_by_default : true, install : true, - install_dir : doc_install_dir) + install_dir : doc_install_dir, + install_tag : 'doc') custom_target('documentation-linkcheck', command : [sphinx, '-W', '-b', 'linkcheck', meson.current_source_dir(), '@OUTPUT@'], diff --git a/Documentation/python-bindings.rst b/Documentation/python-bindings.rst index bac5cd34..94712238 100644 --- a/Documentation/python-bindings.rst +++ b/Documentation/python-bindings.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: documentation-contents.rst + .. _python-bindings: Python Bindings for libcamera @@ -17,13 +19,13 @@ chapter lists the differences. Mostly these differences fall under two categories: 1. Differences caused by the inherent differences between C++ and Python. -These differences are usually caused by the use of threads or differences in -C++ vs Python memory management. + These differences are usually caused by the use of threads or differences in + C++ vs Python memory management. 2. Differences caused by the code being work-in-progress. It's not always -trivial to create a binding in a satisfying way, and the current bindings -contain simplified versions of the C++ API just to get forward. These -differences are expected to eventually go away. + trivial to create a binding in a satisfying way, and the current bindings + contain simplified versions of the C++ API just to get forward. These + differences are expected to eventually go away. Coding Style ------------ diff --git a/Documentation/sensor_driver_requirements.rst b/Documentation/sensor_driver_requirements.rst index 0e516b34..fb4269d0 100644 --- a/Documentation/sensor_driver_requirements.rst +++ b/Documentation/sensor_driver_requirements.rst @@ -1,5 +1,7 @@ .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. include:: documentation-contents.rst + .. _sensor-driver-requirements: Sensor Driver Requirements diff --git a/Documentation/software-isp-benchmarking.rst b/Documentation/software-isp-benchmarking.rst new file mode 100644 index 00000000..9c2a409b --- /dev/null +++ b/Documentation/software-isp-benchmarking.rst @@ -0,0 +1,79 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +.. include:: documentation-contents.rst + +.. _software-isp-benchmarking: + +Software ISP benchmarking +========================= + +The Software ISP is particularly sensitive to performance regressions therefore +it is a good idea to always benchmark the Software ISP before and after making +changes to it and ensure that there are no performance regressions. + +DebayerCpu class builtin benchmark +---------------------------------- + +The DebayerCpu class has a builtin benchmark. This benchmark measures the time +spent on processing (collecting statistics and debayering) only, it does not +measure the time spent on capturing or outputting the frames. + +The builtin benchmark always runs. So this can be used by simply running "cam" +or "qcam" with a pipeline using the Software ISP. + +When it runs it will skip measuring the first 30 frames to allow the caches and +the CPU temperature (turbo-ing) to warm-up and then it measures 30 fps and shows +the total and per frame processing time using an info level log message: + +.. code-block:: text + + INFO Debayer debayer_cpu.cpp:907 Processed 30 frames in 244317us, 8143 us/frame + +To get stable measurements it is advised to disable any other processes which +may cause significant CPU usage (e.g. disable wifi, bluetooth and browsers). +When possible it is also advisable to disable CPU turbo-ing and +frequency-scaling. + +For example when benchmarking on a Lenovo ThinkPad X1 Yoga Gen 8, with the +charger plugged in, the CPU can be fixed to run at 2 GHz using: + +.. code-block:: shell + + sudo x86_energy_perf_policy --turbo-enable 0 + sudo cpupower frequency-set -d 2GHz -u 2GHz + +with these settings the builtin bench reports a processing time of ~7.8ms/frame +on this laptop for FHD SGRBG10 (unpacked) bayer data. + +Measuring power consumption +--------------------------- + +Since the Software ISP is often used on mobile devices it is also important to +measure power consumption and ensure that that does not regress. + +For example to measure power consumption on a Lenovo ThinkPad X1 Yoga Gen 8 it +needs to be running on battery and it should be configured with its +platform-profile (/sys/firmware/acpi/platform_profile) set to balanced and with +its default turbo and frequency-scaling behavior to match real world usage. + +Then start qcam to capture a FHD picture at 30 fps and position the qcam window +so that it is fully visible. After this run the following command to monitor the +power consumption: + +.. code-block:: shell + + watch -n 10 cat /sys/class/power_supply/BAT0/power_now /sys/class/hwmon/hwmon6/fan?_input + +Note this not only measures the power consumption in µW it also monitors the +speed of this laptop's 2 fans. This is important because depending on the +ambient temperature the 2 fans may spin up while testing and this will cause an +additional power consumption of approx. 0.5 W messing up the measurement. + +After starting qcam + the watch command let the laptop sit without using it for +2 minutes for the readings to stabilize. Then check that the fans have not +turned on and manually take a couple of consecutive power readings and average +these. + +On the example Lenovo ThinkPad X1 Yoga Gen 8 laptop this results in a measured +power consumption of approx. 13 W while running qcam versus approx. 4-5 W while +setting idle with its OLED panel on. diff --git a/Documentation/theme/static/css/theme.css b/Documentation/theme/static/css/theme.css index d4274ea6..a6d43195 100644 --- a/Documentation/theme/static/css/theme.css +++ b/Documentation/theme/static/css/theme.css @@ -283,9 +283,13 @@ div#signature { font-size: 12px; } -#libcamera div.toctree-wrapper { +#licensing div.toctree-wrapper { height: 0px; margin: 0px; padding: 0px; visibility: hidden; } + +.documentation-nav { + display: none; +} diff --git a/Documentation/thread-safety.dox b/Documentation/thread-safety.dox new file mode 100644 index 00000000..df4c457c --- /dev/null +++ b/Documentation/thread-safety.dox @@ -0,0 +1,44 @@ +/** + * \page thread-safety Reentrancy and Thread-Safety + * + * Through the documentation, several terms are used to define how classes and + * their member functions can be used from multiple threads. + * + * - A **reentrant** function may be called simultaneously from multiple + * threads if and only if each invocation uses a different instance of the + * class. This is the default for all member functions not explictly marked + * otherwise. + * + * - \anchor thread-safe A **thread-safe** function may be called + * simultaneously from multiple threads on the same instance of a class. A + * thread-safe function is thus reentrant. Thread-safe functions may also be + * called simultaneously with any other reentrant function of the same class + * on the same instance. + * + * \internal + * - \anchor thread-bound A **thread-bound** function may be called only from + * the thread that the class instances lives in (see section \ref + * thread-objects). For instances of classes that do not derive from the + * Object class, this is the thread in which the instance was created. A + * thread-bound function is not thread-safe, and may or may not be reentrant. + * \endinternal + * + * Neither reentrancy nor thread-safety, in this context, mean that a function + * may be called simultaneously from the same thread, for instance from a + * callback invoked by the function. This may deadlock and isn't allowed unless + * separately documented. + * + * \if internal + * A class is defined as reentrant, thread-safe or thread-bound if all its + * member functions are reentrant, thread-safe or thread-bound respectively. + * \else + * A class is defined as reentrant or thread-safe if all its member functions + * are reentrant or thread-safe respectively. + * \endif + * Some member functions may additionally be documented as having additional + * thread-related attributes. + * + * Most classes are reentrant but not thread-safe, as making them fully + * thread-safe would incur locking costs considered prohibitive for the + * expected use cases. + */ |