1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
|
/* SPDX-License-Identifier: LGPL-2.1-or-later */
/*
* Copyright (C) 2020, Google Inc.
*
* ipu3.cpp - IPU3 Image Processing Algorithms
*/
#include <stdint.h>
#include <sys/mman.h>
#include <linux/intel-ipu3.h>
#include <linux/v4l2-controls.h>
#include <libcamera/base/log.h>
#include <libcamera/buffer.h>
#include <libcamera/control_ids.h>
#include <libcamera/ipa/ipa_interface.h>
#include <libcamera/ipa/ipa_module_info.h>
#include <libcamera/ipa/ipu3_ipa_interface.h>
#include <libcamera/request.h>
#include "libcamera/internal/buffer.h"
#include "ipu3_agc.h"
#include "ipu3_awb.h"
#include "libipa/camera_sensor_helper.h"
static constexpr uint32_t kMaxCellWidthPerSet = 160;
static constexpr uint32_t kMaxCellHeightPerSet = 56;
namespace libcamera {
LOG_DEFINE_CATEGORY(IPAIPU3)
namespace ipa::ipu3 {
class IPAIPU3 : public IPAIPU3Interface
{
public:
int init(const IPASettings &settings) override;
int start() override;
void stop() override {}
int configure(const IPAConfigInfo &configInfo) override;
void mapBuffers(const std::vector<IPABuffer> &buffers) override;
void unmapBuffers(const std::vector<unsigned int> &ids) override;
void processEvent(const IPU3Event &event) override;
private:
void processControls(unsigned int frame, const ControlList &controls);
void fillParams(unsigned int frame, ipu3_uapi_params *params);
void parseStatistics(unsigned int frame,
int64_t frameTimestamp,
const ipu3_uapi_stats_3a *stats);
void setControls(unsigned int frame);
void calculateBdsGrid(const Size &bdsOutputSize);
std::map<unsigned int, MappedFrameBuffer> buffers_;
ControlInfoMap ctrls_;
IPACameraSensorInfo sensorInfo_;
/* Camera sensor controls. */
uint32_t defVBlank_;
uint32_t exposure_;
uint32_t minExposure_;
uint32_t maxExposure_;
uint32_t gain_;
uint32_t minGain_;
uint32_t maxGain_;
/* Interface to the AWB algorithm */
std::unique_ptr<IPU3Awb> awbAlgo_;
/* Interface to the AEC/AGC algorithm */
std::unique_ptr<IPU3Agc> agcAlgo_;
/* Interface to the Camera Helper */
std::unique_ptr<CameraSensorHelper> camHelper_;
/* Local parameter storage */
struct ipu3_uapi_params params_;
struct ipu3_uapi_grid_config bdsGrid_;
};
int IPAIPU3::init(const IPASettings &settings)
{
camHelper_ = CameraSensorHelperFactory::create(settings.sensorModel);
if (camHelper_ == nullptr) {
LOG(IPAIPU3, Error) << "Failed to create camera sensor helper for " << settings.sensorModel;
return -ENODEV;
}
return 0;
}
int IPAIPU3::start()
{
setControls(0);
return 0;
}
/**
* This method calculates a grid for the AWB algorithm in the IPU3 firmware.
* Its input is the BDS output size calculated in the ImgU.
* It is limited for now to the simplest method: find the lesser error
* with the width/height and respective log2 width/height of the cells.
*
* \todo The frame is divided into cells which can be 8x8 => 128x128.
* As a smaller cell improves the algorithm precision, adapting the
* x_start and y_start parameters of the grid would provoke a loss of
* some pixels but would also result in more accurate algorithms.
*/
void IPAIPU3::calculateBdsGrid(const Size &bdsOutputSize)
{
uint32_t minError = std::numeric_limits<uint32_t>::max();
Size best;
Size bestLog2;
bdsGrid_ = {};
for (uint32_t widthShift = 3; widthShift <= 7; ++widthShift) {
uint32_t width = std::min(kMaxCellWidthPerSet,
bdsOutputSize.width >> widthShift);
width = width << widthShift;
for (uint32_t heightShift = 3; heightShift <= 7; ++heightShift) {
int32_t height = std::min(kMaxCellHeightPerSet,
bdsOutputSize.height >> heightShift);
height = height << heightShift;
uint32_t error = std::abs(static_cast<int>(width - bdsOutputSize.width))
+ std::abs(static_cast<int>(height - bdsOutputSize.height));
if (error > minError)
continue;
minError = error;
best.width = width;
best.height = height;
bestLog2.width = widthShift;
bestLog2.height = heightShift;
}
}
bdsGrid_.width = best.width >> bestLog2.width;
bdsGrid_.block_width_log2 = bestLog2.width;
bdsGrid_.height = best.height >> bestLog2.height;
bdsGrid_.block_height_log2 = bestLog2.height;
LOG(IPAIPU3, Debug) << "Best grid found is: ("
<< (int)bdsGrid_.width << " << " << (int)bdsGrid_.block_width_log2 << ") x ("
<< (int)bdsGrid_.height << " << " << (int)bdsGrid_.block_height_log2 << ")";
}
int IPAIPU3::configure(const IPAConfigInfo &configInfo)
{
if (configInfo.entityControls.empty()) {
LOG(IPAIPU3, Error) << "No controls provided";
return -ENODATA;
}
sensorInfo_ = configInfo.sensorInfo;
ctrls_ = configInfo.entityControls.at(0);
const auto itExp = ctrls_.find(V4L2_CID_EXPOSURE);
if (itExp == ctrls_.end()) {
LOG(IPAIPU3, Error) << "Can't find exposure control";
return -EINVAL;
}
const auto itGain = ctrls_.find(V4L2_CID_ANALOGUE_GAIN);
if (itGain == ctrls_.end()) {
LOG(IPAIPU3, Error) << "Can't find gain control";
return -EINVAL;
}
const auto itVBlank = ctrls_.find(V4L2_CID_VBLANK);
if (itVBlank == ctrls_.end()) {
LOG(IPAIPU3, Error) << "Can't find VBLANK control";
return -EINVAL;
}
minExposure_ = std::max(itExp->second.min().get<int32_t>(), 1);
maxExposure_ = itExp->second.max().get<int32_t>();
exposure_ = minExposure_;
minGain_ = std::max(itGain->second.min().get<int32_t>(), 1);
maxGain_ = itGain->second.max().get<int32_t>();
gain_ = minGain_;
defVBlank_ = itVBlank->second.def().get<int32_t>();
params_ = {};
calculateBdsGrid(configInfo.bdsOutputSize);
awbAlgo_ = std::make_unique<IPU3Awb>();
awbAlgo_->initialise(params_, configInfo.bdsOutputSize, bdsGrid_);
agcAlgo_ = std::make_unique<IPU3Agc>();
agcAlgo_->initialise(bdsGrid_, sensorInfo_);
return 0;
}
void IPAIPU3::mapBuffers(const std::vector<IPABuffer> &buffers)
{
for (const IPABuffer &buffer : buffers) {
const FrameBuffer fb(buffer.planes);
buffers_.emplace(buffer.id,
MappedFrameBuffer(&fb, PROT_READ | PROT_WRITE));
}
}
void IPAIPU3::unmapBuffers(const std::vector<unsigned int> &ids)
{
for (unsigned int id : ids) {
auto it = buffers_.find(id);
if (it == buffers_.end())
continue;
buffers_.erase(it);
}
}
void IPAIPU3::processEvent(const IPU3Event &event)
{
switch (event.op) {
case EventProcessControls: {
processControls(event.frame, event.controls);
break;
}
case EventStatReady: {
auto it = buffers_.find(event.bufferId);
if (it == buffers_.end()) {
LOG(IPAIPU3, Error) << "Could not find stats buffer!";
return;
}
Span<uint8_t> mem = it->second.maps()[0];
const ipu3_uapi_stats_3a *stats =
reinterpret_cast<ipu3_uapi_stats_3a *>(mem.data());
parseStatistics(event.frame, event.frameTimestamp, stats);
break;
}
case EventFillParams: {
auto it = buffers_.find(event.bufferId);
if (it == buffers_.end()) {
LOG(IPAIPU3, Error) << "Could not find param buffer!";
return;
}
Span<uint8_t> mem = it->second.maps()[0];
ipu3_uapi_params *params =
reinterpret_cast<ipu3_uapi_params *>(mem.data());
fillParams(event.frame, params);
break;
}
default:
LOG(IPAIPU3, Error) << "Unknown event " << event.op;
break;
}
}
void IPAIPU3::processControls([[maybe_unused]] unsigned int frame,
[[maybe_unused]] const ControlList &controls)
{
/* \todo Start processing for 'frame' based on 'controls'. */
}
void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params)
{
if (agcAlgo_->updateControls())
awbAlgo_->updateWbParameters(params_, agcAlgo_->gamma());
*params = params_;
IPU3Action op;
op.op = ActionParamFilled;
queueFrameAction.emit(frame, op);
}
void IPAIPU3::parseStatistics(unsigned int frame,
[[maybe_unused]] int64_t frameTimestamp,
[[maybe_unused]] const ipu3_uapi_stats_3a *stats)
{
ControlList ctrls(controls::controls);
double gain = camHelper_->gain(gain_);
agcAlgo_->process(stats, exposure_, gain);
gain_ = camHelper_->gainCode(gain);
awbAlgo_->calculateWBGains(stats);
if (agcAlgo_->updateControls())
setControls(frame);
/* \todo Use VBlank value calculated from each frame exposure. */
int64_t frameDuration = sensorInfo_.lineLength * (defVBlank_ + sensorInfo_.outputSize.height) /
(sensorInfo_.pixelRate / 1e6);
ctrls.set(controls::FrameDuration, frameDuration);
IPU3Action op;
op.op = ActionMetadataReady;
op.controls = ctrls;
queueFrameAction.emit(frame, op);
}
|