summaryrefslogtreecommitdiff
path: root/LICENSES/Apache-2.0.txt
blob: d645695673349e3947e8e5ae42332d0ac3164cd7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202

                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/

   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION

   1. Definitions.

      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.

      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.

      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.

      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.

      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.

      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.

      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).

      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.

      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."

      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.

   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.

   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.

   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:

      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and

      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and

      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and

      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.

      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.

   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.

   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.

   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.

   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.

   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.

   END OF TERMS AND CONDITIONS

   APPENDIX: How to apply the Apache License to your work.

      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "[]"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.

   Copyright [yyyy] [name of copyright owner]

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
> 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
/* SPDX-License-Identifier: LGPL-2.1-or-later */
/*
 * Copyright (C) 2019-2023, Raspberry Pi Ltd
 *
 * pipeline_base.cpp - Pipeline handler base class for Raspberry Pi devices
 */

#include "pipeline_base.h"

#include <chrono>

#include <linux/media-bus-format.h>
#include <linux/videodev2.h>

#include <libcamera/base/file.h>
#include <libcamera/base/utils.h>

#include <libcamera/formats.h>
#include <libcamera/logging.h>
#include <libcamera/property_ids.h>

#include "libcamera/internal/camera_lens.h"
#include "libcamera/internal/ipa_manager.h"
#include "libcamera/internal/v4l2_subdevice.h"

using namespace std::chrono_literals;

namespace libcamera {

using namespace RPi;

LOG_DEFINE_CATEGORY(RPI)

using StreamFlag = RPi::Stream::StreamFlag;

namespace {

constexpr unsigned int defaultRawBitDepth = 12;

PixelFormat mbusCodeToPixelFormat(unsigned int mbus_code,
				  BayerFormat::Packing packingReq)
{
	BayerFormat bayer = BayerFormat::fromMbusCode(mbus_code);

	ASSERT(bayer.isValid());

	bayer.packing = packingReq;
	PixelFormat pix = bayer.toPixelFormat();

	/*
	 * Not all formats (e.g. 8-bit or 16-bit Bayer formats) can have packed
	 * variants. So if the PixelFormat returns as invalid, use the non-packed
	 * conversion instead.
	 */
	if (!pix.isValid()) {
		bayer.packing = BayerFormat::Packing::None;
		pix = bayer.toPixelFormat();
	}

	return pix;
}

bool isMonoSensor(std::unique_ptr<CameraSensor> &sensor)
{
	unsigned int mbusCode = sensor->mbusCodes()[0];
	const BayerFormat &bayer = BayerFormat::fromMbusCode(mbusCode);

	return bayer.order == BayerFormat::Order::MONO;
}

const std::vector<ColorSpace> validColorSpaces = {
	ColorSpace::Sycc,
	ColorSpace::Smpte170m,
	ColorSpace::Rec709
};

std::optional<ColorSpace> findValidColorSpace(const ColorSpace &colourSpace)
{
	for (auto cs : validColorSpaces) {
		if (colourSpace.primaries == cs.primaries &&
		    colourSpace.transferFunction == cs.transferFunction)
			return cs;
	}

	return std::nullopt;
}

} /* namespace */

/*
 * Raspberry Pi drivers expect the following colour spaces:
 * - V4L2_COLORSPACE_RAW for raw streams.
 * - One of V4L2_COLORSPACE_JPEG, V4L2_COLORSPACE_SMPTE170M, V4L2_COLORSPACE_REC709 for
 *   non-raw streams. Other fields such as transfer function, YCbCr encoding and
 *   quantisation are not used.
 *
 * The libcamera colour spaces that we wish to use corresponding to these are therefore:
 * - ColorSpace::Raw for V4L2_COLORSPACE_RAW
 * - ColorSpace::Sycc for V4L2_COLORSPACE_JPEG
 * - ColorSpace::Smpte170m for V4L2_COLORSPACE_SMPTE170M
 * - ColorSpace::Rec709 for V4L2_COLORSPACE_REC709
 */
CameraConfiguration::Status RPiCameraConfiguration::validateColorSpaces([[maybe_unused]] ColorSpaceFlags flags)
{
	Status status = Valid;
	yuvColorSpace_.reset();

	for (auto cfg : config_) {
		/* First fix up raw streams to have the "raw" colour space. */
		if (PipelineHandlerBase::isRaw(cfg.pixelFormat)) {
			/* If there was no value here, that doesn't count as "adjusted". */
			if (cfg.colorSpace && cfg.colorSpace != ColorSpace::Raw)
				status = Adjusted;
			cfg.colorSpace = ColorSpace::Raw;
			continue;
		}

		/* Next we need to find our shared colour space. The first valid one will do. */
		if (cfg.colorSpace && !yuvColorSpace_)
			yuvColorSpace_ = findValidColorSpace(cfg.colorSpace.value());
	}

	/* If no colour space was given anywhere, choose sYCC. */
	if (!yuvColorSpace_)
		yuvColorSpace_ = ColorSpace::Sycc;

	/* Note the version of this that any RGB streams will have to use. */
	rgbColorSpace_ = yuvColorSpace_;
	rgbColorSpace_->ycbcrEncoding = ColorSpace::YcbcrEncoding::None;
	rgbColorSpace_->range = ColorSpace::Range::Full;

	/* Go through the streams again and force everyone to the same colour space. */
	for (auto cfg : config_) {
		if (cfg.colorSpace == ColorSpace::Raw)
			continue;

		if (PipelineHandlerBase::isYuv(cfg.pixelFormat) && cfg.colorSpace != yuvColorSpace_) {
			/* Again, no value means "not adjusted". */
			if (cfg.colorSpace)
				status = Adjusted;
			cfg.colorSpace = yuvColorSpace_;
		}
		if (PipelineHandlerBase::isRgb(cfg.pixelFormat) && cfg.colorSpace != rgbColorSpace_) {
			/* Be nice, and let the YUV version count as non-adjusted too. */
			if (cfg.colorSpace && cfg.colorSpace != yuvColorSpace_)
				status = Adjusted;
			cfg.colorSpace = rgbColorSpace_;
		}
	}

	return status;
}

CameraConfiguration::Status RPiCameraConfiguration::validate()
{
	Status status = Valid;

	if (config_.empty())
		return Invalid;

	/*
	 * Make sure that if a sensor configuration has been requested it
	 * is valid.
	 */
	if (sensorConfig && !sensorConfig->isValid()) {
		LOG(RPI, Error) << "Invalid sensor configuration request";
		return Invalid;
	}

	status = validateColorSpaces(ColorSpaceFlag::StreamsShareColorSpace);

	/*
	 * Validate the requested transform against the sensor capabilities and
	 * rotation and store the final combined transform that configure() will
	 * need to apply to the sensor to save us working it out again.
	 */
	Orientation requestedOrientation = orientation;
	combinedTransform_ = data_->sensor_->computeTransform(&orientation);
	if (orientation != requestedOrientation)
		status = Adjusted;

	rawStreams_.clear();
	outStreams_.clear();

	for (const auto &[index, cfg] : utils::enumerate(config_)) {
		if (PipelineHandlerBase::isRaw(cfg.pixelFormat))
			rawStreams_.emplace_back(index, &cfg);
		else
			outStreams_.emplace_back(index, &cfg);
	}

	/* Sort the streams so the highest resolution is first. */
	std::sort(rawStreams_.begin(), rawStreams_.end(),
		  [](auto &l, auto &r) { return l.cfg->size > r.cfg->size; });

	std::sort(outStreams_.begin(), outStreams_.end(),
		  [](auto &l, auto &r) { return l.cfg->size > r.cfg->size; });

	/* Compute the sensor's format then do any platform specific fixups. */
	unsigned int bitDepth;
	Size sensorSize;

	if (sensorConfig) {
		/* Use the application provided sensor configuration. */
		bitDepth = sensorConfig->bitDepth;
		sensorSize = sensorConfig->outputSize;
	} else if (!rawStreams_.empty()) {
		/* Use the RAW stream format and size. */
		BayerFormat bayerFormat = BayerFormat::fromPixelFormat(rawStreams_[0].cfg->pixelFormat);
		bitDepth = bayerFormat.bitDepth;
		sensorSize = rawStreams_[0].cfg->size;
	} else {
		bitDepth = defaultRawBitDepth;
		sensorSize = outStreams_[0].cfg->size;
	}

	sensorFormat_ = data_->findBestFormat(sensorSize, bitDepth);

	/*
	 * If a sensor configuration has been requested, it should apply
	 * without modifications.
	 */
	if (sensorConfig) {
		BayerFormat bayer = BayerFormat::fromMbusCode(sensorFormat_.mbus_code);

		if (bayer.bitDepth != sensorConfig->bitDepth ||
		    sensorFormat_.size != sensorConfig->outputSize) {
			LOG(RPI, Error) << "Invalid sensor configuration: "
					<< "bitDepth/size mismatch";
			return Invalid;
		}
	}

	/* Start with some initial generic RAW stream adjustments. */
	for (auto &raw : rawStreams_) {
		StreamConfiguration *rawStream = raw.cfg;

		/* Adjust the RAW stream to match the computed sensor format. */
		BayerFormat sensorBayer = BayerFormat::fromMbusCode(sensorFormat_.mbus_code);

		/*
		 * Some sensors change their Bayer order when they are h-flipped
		 * or v-flipped, according to the transform. If this one does, we
		 * must advertise the transformed Bayer order in the raw stream.
		 * Note how we must fetch the "native" (i.e. untransformed) Bayer
		 * order, because the sensor may currently be flipped!
		 */
		if (data_->flipsAlterBayerOrder_) {
			sensorBayer.order = data_->nativeBayerOrder_;
			sensorBayer = sensorBayer.transform(combinedTransform_);
		}

		/* Apply the sensor adjusted Bayer order to the user request. */
		BayerFormat cfgBayer = BayerFormat::fromPixelFormat(rawStream->pixelFormat);
		cfgBayer.order = sensorBayer.order;

		if (rawStream->pixelFormat != cfgBayer.toPixelFormat()) {
			rawStream->pixelFormat = cfgBayer.toPixelFormat();
			status = Adjusted;
		}
	}

	/* Do any platform specific fixups. */
	Status st = data_->platformValidate(this);
	if (st == Invalid)
		return Invalid;
	else if (st == Adjusted)
		status = Adjusted;

	/* Further fixups on the RAW streams. */
	for (auto &raw : rawStreams_) {
		int ret = raw.dev->tryFormat(&raw.format);
		if (ret)
			return Invalid;

		if (RPi::PipelineHandlerBase::updateStreamConfig(raw.cfg, raw.format))
			status = Adjusted;
	}

	/* Further fixups on the ISP output streams. */
	for (auto &out : outStreams_) {

		/*
		 * We want to send the associated YCbCr info through to the driver.
		 *
		 * But for RGB streams, the YCbCr info gets overwritten on the way back
		 * so we must check against what the stream cfg says, not what we actually
		 * requested (which carefully included the YCbCr info)!
		 */
		out.format.colorSpace = yuvColorSpace_;

		LOG(RPI, Debug)
			<< "Try color space " << ColorSpace::toString(out.cfg->colorSpace);

		int ret = out.dev->tryFormat(&out.format);
		if (ret)
			return Invalid;

		if (RPi::PipelineHandlerBase::updateStreamConfig(out.cfg, out.format))
			status = Adjusted;
	}

	return status;
}

bool PipelineHandlerBase::isRgb(const PixelFormat &pixFmt)
{
	const PixelFormatInfo &info = PixelFormatInfo::info(pixFmt);
	return info.colourEncoding == PixelFormatInfo::ColourEncodingRGB;
}

bool PipelineHandlerBase::isYuv(const PixelFormat &pixFmt)
{
	/* The code below would return true for raw mono streams, so weed those out first. */
	if (PipelineHandlerBase::isRaw(pixFmt))
		return false;

	const PixelFormatInfo &info = PixelFormatInfo::info(pixFmt);
	return info.colourEncoding == PixelFormatInfo::ColourEncodingYUV;
}

bool PipelineHandlerBase::isRaw(const PixelFormat &pixFmt)
{
	/* This test works for both Bayer and raw mono formats. */
	return BayerFormat::fromPixelFormat(pixFmt).isValid();
}

/*
 * Adjust a StreamConfiguration fields to match a video device format.
 * Returns true if the StreamConfiguration has been adjusted.
 */
bool PipelineHandlerBase::updateStreamConfig(StreamConfiguration *stream,
					     const V4L2DeviceFormat &format)
{
	const PixelFormat &pixFormat = format.fourcc.toPixelFormat();
	bool adjusted = false;

	if (stream->pixelFormat != pixFormat || stream->size != format.size) {
		stream->pixelFormat = pixFormat;
		stream->size = format.size;
		adjusted = true;
	}

	if (stream->colorSpace != format.colorSpace) {
		stream->colorSpace = format.colorSpace;
		adjusted = true;
		LOG(RPI, Debug)
			<< "Color space changed from "
			<< ColorSpace::toString(stream->colorSpace) << " to "
			<< ColorSpace::toString(format.colorSpace);
	}

	stream->stride = format.planes[0].bpl;
	stream->frameSize = format.planes[0].size;

	return adjusted;
}

/*
 * Populate and return a video device format using a StreamConfiguration. */
V4L2DeviceFormat PipelineHandlerBase::toV4L2DeviceFormat(const V4L2VideoDevice *dev,
							 const StreamConfiguration *stream)
{
	V4L2DeviceFormat deviceFormat;

	const PixelFormatInfo &info = PixelFormatInfo::info(stream->pixelFormat);
	deviceFormat.planesCount = info.numPlanes();
	deviceFormat.fourcc = dev->toV4L2PixelFormat(stream->pixelFormat);
	deviceFormat.size = stream->size;
	deviceFormat.colorSpace = stream->colorSpace;

	return deviceFormat;
}

V4L2DeviceFormat PipelineHandlerBase::toV4L2DeviceFormat(const V4L2VideoDevice *dev,
							 const V4L2SubdeviceFormat &format,
							 BayerFormat::Packing packingReq)
{
	unsigned int mbus_code = format.mbus_code;
	const PixelFormat pix = mbusCodeToPixelFormat(mbus_code, packingReq);
	V4L2DeviceFormat deviceFormat;

	deviceFormat.fourcc = dev->toV4L2PixelFormat(pix);
	deviceFormat.size = format.size;
	deviceFormat.colorSpace = format.colorSpace;
	return deviceFormat;
}

std::unique_ptr<CameraConfiguration>
PipelineHandlerBase::generateConfiguration(Camera *camera, Span<const StreamRole> roles)
{
	CameraData *data = cameraData(camera);
	std::unique_ptr<CameraConfiguration> config =
		std::make_unique<RPiCameraConfiguration>(data);
	V4L2SubdeviceFormat sensorFormat;
	unsigned int bufferCount;
	PixelFormat pixelFormat;
	V4L2VideoDevice::Formats fmts;
	Size size;
	std::optional<ColorSpace> colorSpace;

	if (roles.empty())
		return config;

	Size sensorSize = data->sensor_->resolution();
	for (const StreamRole role : roles) {
		switch (role) {
		case StreamRole::Raw:
			size = sensorSize;
			sensorFormat = data->findBestFormat(size, defaultRawBitDepth);
			pixelFormat = mbusCodeToPixelFormat(sensorFormat.mbus_code,
							    BayerFormat::Packing::CSI2);
			ASSERT(pixelFormat.isValid());
			colorSpace = ColorSpace::Raw;
			bufferCount = 2;
			break;

		case StreamRole::StillCapture:
			fmts = data->ispFormats();
			pixelFormat = formats::YUV420;
			/*
			 * Still image codecs usually expect the sYCC color space.
			 * Even RGB codecs will be fine as the RGB we get with the
			 * sYCC color space is the same as sRGB.
			 */
			colorSpace = ColorSpace::Sycc;
			/* Return the largest sensor resolution. */
			size = sensorSize;
			bufferCount = 1;
			break;

		case StreamRole::VideoRecording:
			/*
			 * The colour denoise algorithm requires the analysis
			 * image, produced by the second ISP output, to be in
			 * YUV420 format. Select this format as the default, to
			 * maximize chances that it will be picked by
			 * applications and enable usage of the colour denoise
			 * algorithm.
			 */
			fmts = data->ispFormats();
			pixelFormat = formats::YUV420;
			/*
			 * Choose a color space appropriate for video recording.
			 * Rec.709 will be a good default for HD resolutions.
			 */
			colorSpace = ColorSpace::Rec709;
			size = { 1920, 1080 };
			bufferCount = 4;
			break;

		case StreamRole::Viewfinder:
			fmts = data->ispFormats();
			pixelFormat = formats::XRGB8888;
			colorSpace = ColorSpace::Sycc;
			size = { 800, 600 };
			bufferCount = 4;
			break;

		default:
			LOG(RPI, Error) << "Requested stream role not supported: "
					<< role;
			return nullptr;
		}

		std::map<PixelFormat, std::vector<SizeRange>> deviceFormats;
		if (role == StreamRole::Raw) {
			/* Translate the MBUS codes to a PixelFormat. */
			for (const auto &format : data->sensorFormats_) {
				PixelFormat pf = mbusCodeToPixelFormat(format.first,
								       BayerFormat::Packing::CSI2);
				if (pf.isValid())
					deviceFormats.emplace(std::piecewise_construct, std::forward_as_tuple(pf),
							      std::forward_as_tuple(format.second.begin(), format.second.end()));
			}
		} else {
			/*
			 * Translate the V4L2PixelFormat to PixelFormat. Note that we
			 * limit the recommended largest ISP output size to match the
			 * sensor resolution.
			 */
			for (const auto &format : fmts) {
				PixelFormat pf = format.first.toPixelFormat();
				if (pf.isValid()) {
					const SizeRange &ispSizes = format.second[0];
					deviceFormats[pf].emplace_back(ispSizes.min, sensorSize,
								       ispSizes.hStep, ispSizes.vStep);
				}
			}
		}

		/* Add the stream format based on the device node used for the use case. */
		StreamFormats formats(deviceFormats);
		StreamConfiguration cfg(formats);
		cfg.size = size;
		cfg.pixelFormat = pixelFormat;
		cfg.colorSpace = colorSpace;
		cfg.bufferCount = bufferCount;
		config->addConfiguration(cfg);
	}

	config->validate();

	return config;
}

int PipelineHandlerBase::configure(Camera *camera, CameraConfiguration *config)
{
	CameraData *data = cameraData(camera);
	int ret;

	/* Start by freeing all buffers and reset the stream states. */
	data->freeBuffers();
	for (auto const stream : data->streams_)
		stream->clearFlags(StreamFlag::External);

	/*
	 * Apply the format on the sensor with any cached transform.
	 *
	 * If the application has provided a sensor configuration apply it
	 * instead of just applying a format.
	 */
	RPiCameraConfiguration *rpiConfig = static_cast<RPiCameraConfiguration *>(config);
	V4L2SubdeviceFormat *sensorFormat = &rpiConfig->sensorFormat_;

	if (rpiConfig->sensorConfig) {
		ret = data->sensor_->applyConfiguration(*rpiConfig->sensorConfig,
							rpiConfig->combinedTransform_,
							sensorFormat);
	} else {
		ret = data->sensor_->setFormat(sensorFormat,
					       rpiConfig->combinedTransform_);
	}
	if (ret)
		return ret;

	/*
	 * Platform specific internal stream configuration. This also assigns
	 * external streams which get configured below.
	 */
	ret = data->platformConfigure(rpiConfig);
	if (ret)
		return ret;

	ipa::RPi::ConfigResult result;
	ret = data->configureIPA(config, &result);
	if (ret) {
		LOG(RPI, Error) << "Failed to configure the IPA: " << ret;
		return ret;
	}

	/*
	 * Set the scaler crop to the value we are using (scaled to native sensor
	 * coordinates).
	 */
	data->scalerCrop_ = data->scaleIspCrop(data->ispCrop_);

	/*
	 * Update the ScalerCropMaximum to the correct value for this camera mode.
	 * For us, it's the same as the "analogue crop".
	 *
	 * \todo Make this property the ScalerCrop maximum value when dynamic
	 * controls are available and set it at validate() time
	 */
	data->properties_.set(properties::ScalerCropMaximum, data->sensorInfo_.analogCrop);

	/* Store the mode sensitivity for the application. */
	data->properties_.set(properties::SensorSensitivity, result.modeSensitivity);

	/* Update the controls that the Raspberry Pi IPA can handle. */
	ControlInfoMap::Map ctrlMap;
	for (auto const &c : result.controlInfo)
		ctrlMap.emplace(c.first, c.second);

	/* Add the ScalerCrop control limits based on the current mode. */
	Rectangle ispMinCrop = data->scaleIspCrop(Rectangle(data->ispMinCropSize_));
	ctrlMap[&controls::ScalerCrop] = ControlInfo(ispMinCrop, data->sensorInfo_.analogCrop, data->scalerCrop_);

	data->controlInfo_ = ControlInfoMap(std::move(ctrlMap), result.controlInfo.idmap());

	/* Setup the Video Mux/Bridge entities. */
	for (auto &[device, link] : data->bridgeDevices_) {
		/*
		 * Start by disabling all the sink pad links on the devices in the
		 * cascade, with the exception of the link connecting the device.
		 */
		for (const MediaPad *p : device->entity()->pads()) {
			if (!(p->flags() & MEDIA_PAD_FL_SINK))
				continue;

			for (MediaLink *l : p->links()) {
				if (l != link)
					l->setEnabled(false);
			}
		}

		/*
		 * Next, enable the entity -> entity links, and setup the pad format.
		 *
		 * \todo Some bridge devices may chainge the media bus code, so we
		 * ought to read the source pad format and propagate it to the sink pad.
		 */
		link->setEnabled(true);
		const MediaPad *sinkPad = link->sink();
		ret = device->setFormat(sinkPad->index(), sensorFormat);
		if (ret) {
			LOG(RPI, Error) << "Failed to set format on " << device->entity()->name()
					<< " pad " << sinkPad->index()
					<< " with format  " << *sensorFormat
					<< ": " << ret;
			return ret;
		}

		LOG(RPI, Debug) << "Configured media link on device " << device->entity()->name()
				<< " on pad " << sinkPad->index();
	}

	return 0;
}

int PipelineHandlerBase::exportFrameBuffers([[maybe_unused]] Camera *camera, libcamera::Stream *stream,
					    std::vector<std::unique_ptr<FrameBuffer>> *buffers)
{
	RPi::Stream *s = static_cast<RPi::Stream *>(stream);
	unsigned int count = stream->configuration().bufferCount;
	int ret = s->dev()->exportBuffers(count, buffers);

	s->setExportedBuffers(buffers);

	return ret;
}

int PipelineHandlerBase::start(Camera *camera, const ControlList *controls)
{
	CameraData *data = cameraData(camera);
	int ret;

	/* Check if a ScalerCrop control was specified. */
	if (controls)
		data->applyScalerCrop(*controls);

	/* Start the IPA. */
	ipa::RPi::StartResult result;
	data->ipa_->start(controls ? *controls : ControlList{ controls::controls },
			  &result);

	/* Apply any gain/exposure settings that the IPA may have passed back. */
	if (!result.controls.empty())
		data->setSensorControls(result.controls);

	/* Configure the number of dropped frames required on startup. */
	data->dropFrameCount_ = data->config_.disableStartupFrameDrops
			      ? 0 : result.dropFrameCount;

	for (auto const stream : data->streams_)
		stream->resetBuffers();

	if (!data->buffersAllocated_) {
		/* Allocate buffers for internal pipeline usage. */
		ret = prepareBuffers(camera);
		if (ret) {
			LOG(RPI, Error) << "Failed to allocate buffers";
			data->freeBuffers();
			stop(camera);
			return ret;
		}
		data->buffersAllocated_ = true;
	}

	/* We need to set the dropFrameCount_ before queueing buffers. */
	ret = queueAllBuffers(camera);
	if (ret) {
		LOG(RPI, Error) << "Failed to queue buffers";
		stop(camera);
		return ret;
	}

	/*
	 * Reset the delayed controls with the gain and exposure values set by
	 * the IPA.
	 */
	data->delayedCtrls_->reset(0);
	data->state_ = CameraData::State::Idle;

	/* Enable SOF event generation. */
	data->frontendDevice()->setFrameStartEnabled(true);

	data->platformStart();

	/* Start all streams. */
	for (auto const stream : data->streams_) {
		ret = stream->dev()->streamOn();
		if (ret) {
			stop(camera);
			return ret;
		}
	}

	return 0;
}

void PipelineHandlerBase::stopDevice(Camera *camera)
{
	CameraData *data = cameraData(camera);

	data->state_ = CameraData::State::Stopped;
	data->platformStop();

	for (auto const stream : data->streams_)
		stream->dev()->streamOff();

	/* Disable SOF event generation. */
	data->frontendDevice()->setFrameStartEnabled(false);

	data->clearIncompleteRequests();

	/* Stop the IPA. */
	data->ipa_->stop();
}

void PipelineHandlerBase::releaseDevice(Camera *camera)
{
	CameraData *data = cameraData(camera);
	data->freeBuffers();
}

int PipelineHandlerBase::queueRequestDevice(Camera *camera, Request *request)
{
	CameraData *data = cameraData(camera);

	if (!data->isRunning())
		return -EINVAL;

	LOG(RPI, Debug) << "queueRequestDevice: New request.";

	/* Push all buffers supplied in the Request to the respective streams. */
	for (auto stream : data->streams_) {
		if (!(stream->getFlags() & StreamFlag::External))
			continue;

		FrameBuffer *buffer = request->findBuffer(stream);
		if (buffer && !stream->getBufferId(buffer)) {
			/*
			 * This buffer is not recognised, so it must have been allocated
			 * outside the v4l2 device. Store it in the stream buffer list
			 * so we can track it.
			 */
			stream->setExportedBuffer(buffer);
		}

		/*
		 * If no buffer is provided by the request for this stream, we
		 * queue a nullptr to the stream to signify that it must use an
		 * internally allocated buffer for this capture request. This
		 * buffer will not be given back to the application, but is used
		 * to support the internal pipeline flow.
		 *
		 * The below queueBuffer() call will do nothing if there are not
		 * enough internal buffers allocated, but this will be handled by
		 * queuing the request for buffers in the RPiStream object.
		 */
		int ret = stream->queueBuffer(buffer);
		if (ret)
			return ret;
	}

	/* Push the request to the back of the queue. */
	data->requestQueue_.push(request);
	data->handleState();

	return 0;
}

int PipelineHandlerBase::registerCamera(std::unique_ptr<RPi::CameraData> &cameraData,
					MediaDevice *frontend, const std::string &frontendName,
					MediaDevice *backend, MediaEntity *sensorEntity)
{
	CameraData *data = cameraData.get();
	int ret;

	data->sensor_ = std::make_unique<CameraSensor>(sensorEntity);
	if (!data->sensor_)
		return -EINVAL;

	if (data->sensor_->init())
		return -EINVAL;

	/* Populate the map of sensor supported formats and sizes. */
	for (auto const mbusCode : data->sensor_->mbusCodes())
		data->sensorFormats_.emplace(mbusCode,
					     data->sensor_->sizes(mbusCode));

	/*
	 * Enumerate all the Video Mux/Bridge devices across the sensor -> Fr
	 * chain. There may be a cascade of devices in this chain!
	 */
	MediaLink *link = sensorEntity->getPadByIndex(0)->links()[0];
	data->enumerateVideoDevices(link, frontendName);

	ipa::RPi::InitResult result;
	if (data->loadIPA(&result)) {
		LOG(RPI, Error) << "Failed to load a suitable IPA library";
		return -EINVAL;
	}

	/*
	 * Setup our delayed control writer with the sensor default
	 * gain and exposure delays. Mark VBLANK for priority write.
	 */
	std::unordered_map<uint32_t, RPi::DelayedControls::ControlParams> params = {
		{ V4L2_CID_ANALOGUE_GAIN, { result.sensorConfig.gainDelay, false } },
		{ V4L2_CID_EXPOSURE, { result.sensorConfig.exposureDelay, false } },
		{ V4L2_CID_HBLANK, { result.sensorConfig.hblankDelay, false } },
		{ V4L2_CID_VBLANK, { result.sensorConfig.vblankDelay, true } }
	};
	data->delayedCtrls_ = std::make_unique<RPi::DelayedControls>(data->sensor_->device(), params);
	data->sensorMetadata_ = result.sensorConfig.sensorMetadata;

	/* Register initial controls that the Raspberry Pi IPA can handle. */
	data->controlInfo_ = std::move(result.controlInfo);

	/* Initialize the camera properties. */
	data->properties_ = data->sensor_->properties();

	/*
	 * The V4L2_CID_NOTIFY_GAINS control, if present, is used to inform the
	 * sensor of the colour gains. It is defined to be a linear gain where
	 * the default value represents a gain of exactly one.
	 */
	auto it = data->sensor_->controls().find(V4L2_CID_NOTIFY_GAINS);
	if (it != data->sensor_->controls().end())
		data->notifyGainsUnity_ = it->second.def().get<int32_t>();

	/*
	 * Set a default value for the ScalerCropMaximum property to show
	 * that we support its use, however, initialise it to zero because
	 * it's not meaningful until a camera mode has been chosen.
	 */
	data->properties_.set(properties::ScalerCropMaximum, Rectangle{});

	/*
	 * We cache two things about the sensor in relation to transforms
	 * (meaning horizontal and vertical flips): if they affect the Bayer
	 * ordering, and what the "native" Bayer order is, when no transforms
	 * are applied.
	 *
	 * If flips are supported verify if they affect the Bayer ordering
	 * and what the "native" Bayer order is, when no transforms are
	 * applied.
	 *
	 * We note that the sensor's cached list of supported formats is
	 * already in the "native" order, with any flips having been undone.
	 */
	const V4L2Subdevice *sensor = data->sensor_->device();
	const struct v4l2_query_ext_ctrl *hflipCtrl = sensor->controlInfo(V4L2_CID_HFLIP);
	if (hflipCtrl) {