diff options
| -rwxr-xr-x | src/py/cam/cam.py | 475 | ||||
| -rw-r--r-- | src/py/cam/cam_kms.py | 183 | ||||
| -rw-r--r-- | src/py/cam/cam_null.py | 47 | ||||
| -rw-r--r-- | src/py/cam/cam_qt.py | 354 | ||||
| -rw-r--r-- | src/py/cam/cam_qtgl.py | 383 | ||||
| -rw-r--r-- | src/py/cam/gl_helpers.py | 74 |
6 files changed, 1516 insertions, 0 deletions
diff --git a/src/py/cam/cam.py b/src/py/cam/cam.py new file mode 100755 index 00000000..012b191c --- /dev/null +++ b/src/py/cam/cam.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2022, Tomi Valkeinen <tomi.valkeinen@ideasonboard.com> + +# \todo Convert ctx and state dicts to proper classes, and move relevant +# functions to those classes. + +import argparse +import binascii +import libcamera as libcam +import os +import sys + + +class CustomAction(argparse.Action): + def __init__(self, option_strings, dest, **kwargs): + super().__init__(option_strings, dest, default={}, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + if len(namespace.camera) == 0: + print(f'Option {option_string} requires a --camera context') + sys.exit(-1) + + if self.type == bool: + values = True + + current = namespace.camera[-1] + + data = getattr(namespace, self.dest) + + if self.nargs == '+': + if current not in data: + data[current] = [] + + data[current] += values + else: + data[current] = values + + +def do_cmd_list(cm): + print('Available cameras:') + + for idx, c in enumerate(cm.cameras): + print(f'{idx + 1}: {c.id}') + + +def do_cmd_list_props(ctx): + camera = ctx['camera'] + + print('Properties for', ctx['id']) + + for name, prop in camera.properties.items(): + print('\t{}: {}'.format(name, prop)) + + +def do_cmd_list_controls(ctx): + camera = ctx['camera'] + + print('Controls for', ctx['id']) + + for name, prop in camera.controls.items(): + print('\t{}: {}'.format(name, prop)) + + +def do_cmd_info(ctx): + camera = ctx['camera'] + + print('Stream info for', ctx['id']) + + roles = [libcam.StreamRole.Viewfinder] + + camconfig = camera.generate_configuration(roles) + if camconfig is None: + raise Exception('Generating config failed') + + for i, stream_config in enumerate(camconfig): + print('\t{}: {}'.format(i, stream_config)) + + formats = stream_config.formats + for fmt in formats.pixel_formats: + print('\t * Pixelformat:', fmt, formats.range(fmt)) + + for size in formats.sizes(fmt): + print('\t -', size) + + +def acquire(ctx): + camera = ctx['camera'] + + camera.acquire() + + +def release(ctx): + camera = ctx['camera'] + + camera.release() + + +def parse_streams(ctx): + streams = [] + + for stream_desc in ctx['opt-stream']: + stream_opts = {'role': libcam.StreamRole.Viewfinder} + + for stream_opt in stream_desc.split(','): + if stream_opt == 0: + continue + + arr = stream_opt.split('=') + if len(arr) != 2: + print('Bad stream option', stream_opt) + sys.exit(-1) + + key = arr[0] + value = arr[1] + + if key in ['width', 'height']: + value = int(value) + elif key == 'role': + rolemap = { + 'still': libcam.StreamRole.StillCapture, + 'raw': libcam.StreamRole.Raw, + 'video': libcam.StreamRole.VideoRecording, + 'viewfinder': libcam.StreamRole.Viewfinder, + } + + role = rolemap.get(value.lower(), None) + + if role is None: + print('Bad stream role', value) + sys.exit(-1) + + value = role + elif key == 'pixelformat': + pass + else: + print('Bad stream option key', key) + sys.exit(-1) + + stream_opts[key] = value + + streams.append(stream_opts) + + return streams + + +def configure(ctx): + camera = ctx['camera'] + + streams = parse_streams(ctx) + + roles = [opts['role'] for opts in streams] + + camconfig = camera.generate_configuration(roles) + if camconfig is None: + raise Exception('Generating config failed') + + for idx, stream_opts in enumerate(streams): + stream_config = camconfig.at(idx) + + if 'width' in stream_opts and 'height' in stream_opts: + stream_config.size = (stream_opts['width'], stream_opts['height']) + + if 'pixelformat' in stream_opts: + stream_config.pixel_format = stream_opts['pixelformat'] + + stat = camconfig.validate() + + if stat == libcam.CameraConfiguration.Status.Invalid: + print('Camera configuration invalid') + exit(-1) + elif stat == libcam.CameraConfiguration.Status.Adjusted: + if ctx['opt-strict-formats']: + print('Adjusting camera configuration disallowed by --strict-formats argument') + exit(-1) + + print('Camera configuration adjusted') + + r = camera.configure(camconfig) + if r != 0: + raise Exception('Configure failed') + + ctx['stream-names'] = {} + ctx['streams'] = [] + + for idx, stream_config in enumerate(camconfig): + stream = stream_config.stream + ctx['streams'].append(stream) + ctx['stream-names'][stream] = 'stream' + str(idx) + print('{}-{}: stream config {}'.format(ctx['id'], ctx['stream-names'][stream], stream.configuration)) + + +def alloc_buffers(ctx): + camera = ctx['camera'] + + allocator = libcam.FrameBufferAllocator(camera) + + for idx, stream in enumerate(ctx['streams']): + ret = allocator.allocate(stream) + if ret < 0: + print('Cannot allocate buffers') + exit(-1) + + allocated = len(allocator.buffers(stream)) + + print('{}-{}: Allocated {} buffers'.format(ctx['id'], ctx['stream-names'][stream], allocated)) + + ctx['allocator'] = allocator + + +def create_requests(ctx): + camera = ctx['camera'] + + ctx['requests'] = [] + + # Identify the stream with the least number of buffers + num_bufs = min([len(ctx['allocator'].buffers(stream)) for stream in ctx['streams']]) + + requests = [] + + for buf_num in range(num_bufs): + request = camera.create_request(ctx['idx']) + + if request is None: + print('Can not create request') + exit(-1) + + for stream in ctx['streams']: + buffers = ctx['allocator'].buffers(stream) + buffer = buffers[buf_num] + + ret = request.add_buffer(stream, buffer) + if ret < 0: + print('Can not set buffer for request') + exit(-1) + + requests.append(request) + + ctx['requests'] = requests + + +def start(ctx): + camera = ctx['camera'] + + camera.start() + + +def stop(ctx): + camera = ctx['camera'] + + camera.stop() + + +def queue_requests(ctx): + camera = ctx['camera'] + + for request in ctx['requests']: + camera.queue_request(request) + ctx['reqs-queued'] += 1 + + del ctx['requests'] + + +def capture_init(contexts): + for ctx in contexts: + acquire(ctx) + + for ctx in contexts: + configure(ctx) + + for ctx in contexts: + alloc_buffers(ctx) + + for ctx in contexts: + create_requests(ctx) + + +def capture_start(contexts): + for ctx in contexts: + start(ctx) + + for ctx in contexts: + queue_requests(ctx) + + +# Called from renderer when there is a libcamera event +def event_handler(state): + cm = state['cm'] + contexts = state['contexts'] + + os.read(cm.efd, 8) + + reqs = cm.get_ready_requests() + + for req in reqs: + ctx = next(ctx for ctx in contexts if ctx['idx'] == req.cookie) + request_handler(state, ctx, req) + + running = any(ctx['reqs-completed'] < ctx['opt-capture'] for ctx in contexts) + return running + + +def request_handler(state, ctx, req): + if req.status != libcam.Request.Status.Complete: + raise Exception('{}: Request failed: {}'.format(ctx['id'], req.status)) + + buffers = req.buffers + + # Compute the frame rate. The timestamp is arbitrarily retrieved from + # the first buffer, as all buffers should have matching timestamps. + ts = buffers[next(iter(buffers))].metadata.timestamp + last = ctx.get('last', 0) + fps = 1000000000.0 / (ts - last) if (last != 0 and (ts - last) != 0) else 0 + ctx['last'] = ts + ctx['fps'] = fps + + for stream, fb in buffers.items(): + stream_name = ctx['stream-names'][stream] + + crcs = [] + if ctx['opt-crc']: + with fb.mmap() as mfb: + plane_crcs = [binascii.crc32(p) for p in mfb.planes] + crcs.append(plane_crcs) + + meta = fb.metadata + + print('{:.6f} ({:.2f} fps) {}-{}: seq {}, bytes {}, CRCs {}' + .format(ts / 1000000000, fps, + ctx['id'], stream_name, + meta.sequence, meta.bytesused, + crcs)) + + if ctx['opt-metadata']: + reqmeta = req.metadata + for ctrl, val in reqmeta.items(): + print(f'\t{ctrl} = {val}') + + if ctx['opt-save-frames']: + with fb.mmap() as mfb: + filename = 'frame-{}-{}-{}.data'.format(ctx['id'], stream_name, ctx['reqs-completed']) + with open(filename, 'wb') as f: + for p in mfb.planes: + f.write(p) + + state['renderer'].request_handler(ctx, req) + + ctx['reqs-completed'] += 1 + + +# Called from renderer when it has finished with a request +def request_prcessed(ctx, req): + camera = ctx['camera'] + + if ctx['reqs-queued'] < ctx['opt-capture']: + req.reuse() + camera.queue_request(req) + ctx['reqs-queued'] += 1 + + +def capture_deinit(contexts): + for ctx in contexts: + stop(ctx) + + for ctx in contexts: + release(ctx) + + +def do_cmd_capture(state): + capture_init(state['contexts']) + + renderer = state['renderer'] + + renderer.setup() + + capture_start(state['contexts']) + + renderer.run() + + capture_deinit(state['contexts']) + + +def main(): + parser = argparse.ArgumentParser() + # global options + parser.add_argument('-l', '--list', action='store_true', help='List all cameras') + parser.add_argument('-c', '--camera', type=int, action='extend', nargs=1, default=[], help='Specify which camera to operate on, by index') + parser.add_argument('-p', '--list-properties', action='store_true', help='List cameras properties') + parser.add_argument('--list-controls', action='store_true', help='List cameras controls') + parser.add_argument('-I', '--info', action='store_true', help='Display information about stream(s)') + parser.add_argument('-R', '--renderer', default='null', help='Renderer (null, kms, qt, qtgl)') + + # per camera options + parser.add_argument('-C', '--capture', nargs='?', type=int, const=1000000, action=CustomAction, help='Capture until interrupted by user or until CAPTURE frames captured') + parser.add_argument('--crc', nargs=0, type=bool, action=CustomAction, help='Print CRC32 for captured frames') + parser.add_argument('--save-frames', nargs=0, type=bool, action=CustomAction, help='Save captured frames to files') + parser.add_argument('--metadata', nargs=0, type=bool, action=CustomAction, help='Print the metadata for completed requests') + parser.add_argument('--strict-formats', type=bool, nargs=0, action=CustomAction, help='Do not allow requested stream format(s) to be adjusted') + parser.add_argument('-s', '--stream', nargs='+', action=CustomAction) + args = parser.parse_args() + + cm = libcam.CameraManager.singleton() + + if args.list: + do_cmd_list(cm) + + contexts = [] + + for cam_idx in args.camera: + camera = next((c for i, c in enumerate(cm.cameras) if i + 1 == cam_idx), None) + + if camera is None: + print('Unable to find camera', cam_idx) + return -1 + + contexts.append({ + 'camera': camera, + 'idx': cam_idx, + 'id': 'cam' + str(cam_idx), + 'reqs-queued': 0, + 'reqs-completed': 0, + 'opt-capture': args.capture.get(cam_idx, False), + 'opt-crc': args.crc.get(cam_idx, False), + 'opt-save-frames': args.save_frames.get(cam_idx, False), + 'opt-metadata': args.metadata.get(cam_idx, False), + 'opt-strict-formats': args.strict_formats.get(cam_idx, False), + 'opt-stream': args.stream.get(cam_idx, ['role=viewfinder']), + }) + + for ctx in contexts: + print('Using camera {} as {}'.format(ctx['camera'].id, ctx['id'])) + + for ctx in contexts: + if args.list_properties: + do_cmd_list_props(ctx) + if args.list_controls: + do_cmd_list_controls(ctx) + if args.info: + do_cmd_info(ctx) + + if args.capture: + + state = { + 'cm': cm, + 'contexts': contexts, + 'event_handler': event_handler, + 'request_prcessed': request_prcessed, + } + + if args.renderer == 'null': + import cam_null + renderer = cam_null.NullRenderer(state) + elif args.renderer == 'kms': + import cam_kms + renderer = cam_kms.KMSRenderer(state) + elif args.renderer == 'qt': + import cam_qt + renderer = cam_qt.QtRenderer(state) + elif args.renderer == 'qtgl': + import cam_qtgl + renderer = cam_qtgl.QtRenderer(state) + else: + print('Bad renderer', args.renderer) + return -1 + + state['renderer'] = renderer + + do_cmd_capture(state) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/py/cam/cam_kms.py b/src/py/cam/cam_kms.py new file mode 100644 index 00000000..ae6be277 --- /dev/null +++ b/src/py/cam/cam_kms.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2022, Tomi Valkeinen <tomi.valkeinen@ideasonboard.com> + +import pykms +import selectors +import sys + +FMT_MAP = { + 'RGB888': pykms.PixelFormat.RGB888, + 'YUYV': pykms.PixelFormat.YUYV, + 'ARGB8888': pykms.PixelFormat.ARGB8888, + 'XRGB8888': pykms.PixelFormat.XRGB8888, +} + + +class KMSRenderer: + def __init__(self, state): + self.state = state + + self.cm = state['cm'] + self.contexts = state['contexts'] + self.running = False + + card = pykms.Card() + + res = pykms.ResourceManager(card) + conn = res.reserve_connector() + crtc = res.reserve_crtc(conn) + mode = conn.get_default_mode() + modeb = mode.to_blob(card) + + req = pykms.AtomicReq(card) + req.add_connector(conn, crtc) + req.add_crtc(crtc, modeb) + r = req.commit_sync(allow_modeset=True) + assert(r == 0) + + self.card = card + self.resman = res + self.crtc = crtc + self.mode = mode + + self.bufqueue = [] + self.current = None + self.next = None + self.cam_2_drm = {} + + # KMS + + def close(self): + req = pykms.AtomicReq(self.card) + for s in self.streams: + req.add_plane(s['plane'], None, None, dst=(0, 0, 0, 0)) + req.commit() + + def add_plane(self, req, stream, fb): + s = next(s for s in self.streams if s['stream'] == stream) + idx = s['idx'] + plane = s['plane'] + + if idx % 2 == 0: + x = 0 + else: + x = self.mode.hdisplay - fb.width + + if idx // 2 == 0: + y = 0 + else: + y = self.mode.vdisplay - fb.height + + req.add_plane(plane, fb, self.crtc, dst=(x, y, fb.width, fb.height)) + + def apply_request(self, drmreq): + + buffers = drmreq['camreq'].buffers + + for stream, fb in buffers.items(): + drmfb = self.cam_2_drm.get(fb, None) + + req = pykms.AtomicReq(self.card) + self.add_plane(req, stream, drmfb) + req.commit() + + def handle_page_flip(self, frame, time): + old = self.current + self.current = self.next + + if len(self.bufqueue) > 0: + self.next = self.bufqueue.pop(0) + else: + self.next = None + + if self.next: + drmreq = self.next + + self.apply_request(drmreq) + + if old: + req = old['camreq'] + ctx = old['camctx'] + self.state['request_prcessed'](ctx, req) + + def queue(self, drmreq): + if not self.next: + self.next = drmreq + self.apply_request(drmreq) + else: + self.bufqueue.append(drmreq) + + # libcamera + + def setup(self): + self.streams = [] + + idx = 0 + for ctx in self.contexts: + for stream in ctx['streams']: + + cfg = stream.configuration + fmt = cfg.pixel_format + fmt = FMT_MAP[fmt] + + plane = self.resman.reserve_generic_plane(self.crtc, fmt) + assert(plane is not None) + + self.streams.append({ + 'idx': idx, + 'stream': stream, + 'plane': plane, + 'fmt': fmt, + 'size': cfg.size, + }) + + for fb in ctx['allocator'].buffers(stream): + w, h = cfg.size + stride = cfg.stride + fd = fb.fd(0) + drmfb = pykms.DmabufFramebuffer(self.card, w, h, fmt, + [fd], [stride], [0]) + self.cam_2_drm[fb] = drmfb + + idx += 1 + + def readdrm(self, fileobj): + for ev in self.card.read_events(): + if ev.type == pykms.DrmEventType.FLIP_COMPLETE: + self.handle_page_flip(ev.seq, ev.time) + + def readcam(self, fd): + self.running = self.state['event_handler'](self.state) + + def readkey(self, fileobj): + sys.stdin.readline() + self.running = False + + def run(self): + print('Capturing...') + + self.running = True + + sel = selectors.DefaultSelector() + sel.register(self.card.fd, selectors.EVENT_READ, self.readdrm) + sel.register(self.cm.efd, selectors.EVENT_READ, self.readcam) + sel.register(sys.stdin, selectors.EVENT_READ, self.readkey) + + print('Press enter to exit') + + while self.running: + events = sel.select() + for key, mask in events: + callback = key.data + callback(key.fileobj) + + print('Exiting...') + + def request_handler(self, ctx, req): + + drmreq = { + 'camctx': ctx, + 'camreq': req, + } + + self.queue(drmreq) diff --git a/src/py/cam/cam_null.py b/src/py/cam/cam_null.py new file mode 100644 index 00000000..a6da9671 --- /dev/null +++ b/src/py/cam/cam_null.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2022, Tomi Valkeinen <tomi.valkeinen@ideasonboard.com> + +import selectors +import sys + + +class NullRenderer: + def __init__(self, state): + self.state = state + + self.cm = state['cm'] + self.contexts = state['contexts'] + + self.running = False + + def setup(self): + pass + + def run(self): + print('Capturing...') + + self.running = True + + sel = selectors.DefaultSelector() + sel.register(self.cm.efd, selectors.EVENT_READ, self.readcam) + sel.register(sys.stdin, selectors.EVENT_READ, self.readkey) + + print('Press enter to exit') + + while self.running: + events = sel.select() + for key, mask in events: + callback = key.data + callback(key.fileobj) + + print('Exiting...') + + def readcam(self, fd): + self.running = self.state['event_handler'](self.state) + + def readkey(self, fileobj): + sys.stdin.readline() + self.running = False + + def request_handler(self, ctx, req): + self.state['request_prcessed'](ctx, req) diff --git a/src/py/cam/cam_qt.py b/src/py/cam/cam_qt.py new file mode 100644 index 00000000..5753f0b2 --- /dev/null +++ b/src/py/cam/cam_qt.py @@ -0,0 +1,354 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2022, Tomi Valkeinen <tomi.valkeinen@ideasonboard.com> +# +# Debayering code from PiCamera documentation + +from io import BytesIO +from numpy.lib.stride_tricks import as_strided +from PIL import Image +from PIL.ImageQt import ImageQt +from PyQt5 import QtCore, QtGui, QtWidgets +import numpy as np +import sys + + +def rgb_to_pix(rgb): + img = Image.frombuffer('RGB', (rgb.shape[1], rgb.shape[0]), rgb) + qim = ImageQt(img).copy() + pix = QtGui.QPixmap.fromImage(qim) + return pix + + +def separate_components(data, r0, g0, g1, b0): + # Now to split the data up into its red, green, and blue components. The + # Bayer pattern of the OV5647 sensor is BGGR. In other words the first + # row contains alternating green/blue elements, the second row contains + # alternating red/green elements, and so on as illustrated below: + # + # GBGBGBGBGBGBGB + # RGRGRGRGRGRGRG + # GBGBGBGBGBGBGB + # RGRGRGRGRGRGRG + # + # Please note that if you use vflip or hflip to change the orientation + # of the capture, you must flip the Bayer pattern accordingly + + rgb = np.zeros(data.shape + (3,), dtype=data.dtype) + rgb[r0[1]::2, r0[0]::2, 0] = data[r0[1]::2, r0[0]::2] # Red + rgb[g0[1]::2, g0[0]::2, 1] = data[g0[1]::2, g0[0]::2] # Green + rgb[g1[1]::2, g1[0]::2, 1] = data[g1[1]::2, g1[0]::2] # Green + rgb[b0[1]::2, b0[0]::2, 2] = data[b0[1]::2, b0[0]::2] # Blue + + return rgb + + +def demosaic(rgb, r0, g0, g1, b0): + # At this point we now have the raw Bayer data with the correct values + # and colors but the data still requires de-mosaicing and + # post-processing. If you wish to do this yourself, end the script here! + # + # Below we present a fairly naive de-mosaic method that simply + # calculates the weighted average of a pixel based on the pixels + # surrounding it. The weighting is provided b0[1] a b0[1]te representation of + # the Bayer filter which we construct first: + + bayer = np.zeros(rgb.shape, dtype=np.uint8) + bayer[r0[1]::2, r0[0]::2, 0] = 1 # Red + bayer[g0[1]::2, g0[0]::2, 1] = 1 # Green + bayer[g1[1]::2, g1[0]::2, 1] = 1 # Green + bayer[b0[1]::2, b0[0]::2, 2] = 1 # Blue + + # Allocate an array to hold our output with the same shape as the input + # data. After this we define the size of window that will be used to + # calculate each weighted average (3x3). Then we pad out the rgb and + # bayer arrays, adding blank pixels at their edges to compensate for the + # size of the window when calculating averages for edge pixels. + + output = np.empty(rgb.shape, dtype=rgb.dtype) + window = (3, 3) + borders = (window[0] - 1, window[1] - 1) + border = (borders[0] // 2, borders[1] // 2) + + # rgb_pad = np.zeros(( + # rgb.shape[0] + borders[0], + # rgb.shape[1] + borders[1], + # rgb.shape[2]), dtype=rgb.dtype) + # rgb_pad[ + # border[0]:rgb_pad.shape[0] - border[0], + # border[1]:rgb_pad.shape[1] - border[1], + # :] = rgb + # rgb = rgb_pad + # + # bayer_pad = np.zeros(( + # bayer.shape[0] + borders[0], + # bayer.shape[1] + borders[1], + # bayer.shape[2]), dtype=bayer.dtype) + # bayer_pad[ + # border[0]:bayer_pad.shape[0] - border[0], + # border[1]:bayer_pad.shape[1] - border[1], + # :] = bayer + # bayer = bayer_pad + + # In numpy >=1.7.0 just use np.pad (version in Raspbian is 1.6.2 at the + # time of writing...) + # + rgb = np.pad(rgb, [ + (border[0], border[0]), + (border[1], border[1]), + (0, 0), + ], 'constant') + bayer = np.pad(bayer, [ + (border[0], border[0]), + (border[1], border[1]), + (0, 0), + ], 'constant') + + # For each plane in the RGB data, we use a nifty numpy trick + # (as_strided) to construct a view over the plane of 3x3 matrices. We do + # the same for the bayer array, then use Einstein summation on each + # (np.sum is simpler, but copies the data so it's slower), and divide + # the results to get our weighted average: + + for plane in range(3): + p = rgb[..., plane] + b = bayer[..., plane] + pview = as_strided(p, shape=( + p.shape[0] - borders[0], + p.shape[1] - borders[1]) + window, strides=p.strides * 2) + bview = as_strided(b, shape=( + b.shape[0] - borders[0], + b.shape[1] - borders[1]) + window, strides=b.strides * 2) + psum = np.einsum('ijkl->ij', pview) + bsum = np.einsum('ijkl->ij', bview) + output[..., plane] = psum // bsum + + return output + + +def to_rgb(fmt, size, data): + w = size[0] + h = size[1] + + if fmt == 'YUYV': + # YUV422 + yuyv = data.reshape((h, w // 2 * 4)) + + # YUV444 + yuv = np.empty((h, w, 3), dtype=np.uint8) + yuv[:, :, 0] = yuyv[:, 0::2] # Y + yuv[:, :, 1] = yuyv[:, 1::4].repeat(2, axis=1) # U + yuv[:, :, 2] = yuyv[:, 3::4].repeat(2, axis=1) # V + + m = np.array([ + [1.0, 1.0, 1.0], + [-0.000007154783816076815, -0.3441331386566162, 1.7720025777816772], + [1.4019975662231445, -0.7141380310058594, 0.00001542569043522235] + ]) + + rgb = np.dot(yuv, m) + rgb[:, :, 0] -= 179.45477266423404 + rgb[:, :, 1] += 135.45870971679688 + rgb[:, :, 2] -= 226.8183044444304 + rgb = rgb.astype(np.uint8) + + elif fmt == 'RGB888': + rgb = data.reshape((h, w, 3)) + rgb[:, :, [0, 1, 2]] = rgb[:, :, [2, 1, 0]] + + elif fmt == 'BGR888': + rgb = data.reshape((h, w, 3)) + + elif fmt in ['ARGB8888', 'XRGB8888']: + rgb = data.reshape((h, w, 4)) + rgb = np.flip(rgb, axis=2) + # drop alpha component + rgb = np.delete(rgb, np.s_[0::4], axis=2) + + elif fmt.startswith('S'): + bayer_pattern = fmt[1:5] + bitspp = int(fmt[5:]) + + # TODO: shifting leaves the lowest bits 0 + if bitspp == 8: + data = data.reshape((h, w)) + data = data.astype(np.uint16) << 8 + elif bitspp in [10, 12]: + data = data.view(np.uint16) + data = data.reshape((h, w)) + data = data << (16 - bitspp) + else: + raise Exception('Bad bitspp:' + str(bitspp)) + + idx = bayer_pattern.find('R') + assert(idx != -1) + r0 = (idx % 2, idx // 2) + + idx = bayer_pattern.find('G') + assert(idx != -1) + g0 = (idx % 2, idx // 2) + + idx = bayer_pattern.find('G', idx + 1) + assert(idx != -1) + g1 = (idx % 2, idx // 2) + + idx = bayer_pattern.find('B') + assert(idx != -1) + b0 = (idx % 2, idx // 2) + + rgb = separate_components(data, r0, g0, g1, b0) + rgb = demosaic(rgb, r0, g0, g1, b0) + rgb = (rgb >> 8).astype(np.uint8) + + else: + rgb = None + + return rgb + + +class QtRenderer: + def __init__(self, state): + self.state = state + + self.cm = state['cm'] + self.contexts = state['contexts'] + + def setup(self): + self.app = QtWidgets.QApplication([]) + + windows = [] + + for ctx in self.contexts: + camera = ctx['camera'] + + for stream in ctx['streams']: + fmt = stream.configuration.pixel_format + size = stream.configuration.size + + window = MainWindow(ctx, stream) + window.setAttribute(QtCore.Qt.WA_ShowWithoutActivating) + window.show() + windows.append(window) + + self.windows = windows + + def run(self): + camnotif = QtCore.QSocketNotifier(self.cm.efd, QtCore.QSocketNotifier.Read) + camnotif.activated.connect(lambda x: self.readcam()) + + keynotif = QtCore.QSocketNotifier(sys.stdin.fileno(), QtCore.QSocketNotifier.Read) + keynotif.activated.connect(lambda x: self.readkey()) + + print('Capturing...') + + self.app.exec() + + print('Exiting...') + + def readcam(self): + running = self.state['event_handler'](self.state) + + if not running: + self.app.quit() + + def readkey(self): + sys.stdin.readline() + self.app.quit() + + def request_handler(self, ctx, req): + buffers = req.buffers + + for stream, fb in buffers.items(): + wnd = next(wnd for wnd in self.windows if wnd.stream == stream) + + wnd.handle_request(stream, fb) + + self.state['request_prcessed'](ctx, req) + + def cleanup(self): + for w in self.windows: + w.close() + + +class MainWindow(QtWidgets.QWidget): + def __init__(self, ctx, stream): + super().__init__() + + self.ctx = ctx + self.stream = stream + + self.label = QtWidgets.QLabel() + + windowLayout = QtWidgets.QHBoxLayout() + self.setLayout(windowLayout) + + windowLayout.addWidget(self.label) + + controlsLayout = QtWidgets.QVBoxLayout() + windowLayout.addLayout(controlsLayout) + + windowLayout.addStretch() + + group = QtWidgets.QGroupBox('Info') + groupLayout = QtWidgets.QVBoxLayout() + group.setLayout(groupLayout) + controlsLayout.addWidget(group) + + lab = QtWidgets.QLabel(ctx['id']) + groupLayout.addWidget(lab) + + self.frameLabel = QtWidgets.QLabel() + groupLayout.addWidget(self.frameLabel) + + group = QtWidgets.QGroupBox('Properties') + groupLayout = QtWidgets.QVBoxLayout() + group.setLayout(groupLayout) + controlsLayout.addWidget(group) + + camera = ctx['camera'] + + for k, v in camera.properties.items(): + lab = QtWidgets.QLabel() + lab.setText(k + ' = ' + str(v)) + groupLayout.addWidget(lab) + + group = QtWidgets.QGroupBox('Controls') + groupLayout = QtWidgets.QVBoxLayout() + group.setLayout(groupLayout) + controlsLayout.addWidget(group) + + for k, (min, max, default) in camera.controls.items(): + lab = QtWidgets.QLabel() + lab.setText('{} = {}/{}/{}'.format(k, min, max, default)) + groupLayout.addWidget(lab) + + controlsLayout.addStretch() + + def buf_to_qpixmap(self, stream, fb): + with fb.mmap() as mfb: + cfg = stream.configuration + w, h = cfg.size + pitch = cfg.stride + + if cfg.pixel_format == 'MJPEG': + img = Image.open(BytesIO(mfb.planes[0])) + qim = ImageQt(img).copy() + pix = QtGui.QPixmap.fromImage(qim) + else: + data = np.array(mfb.planes[0], dtype=np.uint8) + rgb = to_rgb(cfg.pixel_format, cfg.size, data) + + if rgb is None: + raise Exception('Format not supported: ' + cfg.pixel_format) + + pix = rgb_to_pix(rgb) + + return pix + + def handle_request(self, stream, fb): + ctx = self.ctx + + pix = self.buf_to_qpixmap(stream, fb) + self.label.setPixmap(pix) + + self.frameLabel.setText('Queued: {}\nDone: {}\nFps: {:.2f}' + .format(ctx['reqs-queued'], ctx['reqs-completed'], ctx['fps'])) diff --git a/src/py/cam/cam_qtgl.py b/src/py/cam/cam_qtgl.py new file mode 100644 index 00000000..8a95994e --- /dev/null +++ b/src/py/cam/cam_qtgl.py @@ -0,0 +1,383 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2022, Tomi Valkeinen <tomi.valkeinen@ideasonboard.com> + +from PyQt5 import QtCore, QtWidgets +from PyQt5.QtCore import Qt + +import math +import numpy as np +import os +import sys + +os.environ['PYOPENGL_PLATFORM'] = 'egl' + +import OpenGL +# OpenGL.FULL_LOGGING = True + +from OpenGL import GL as gl +from OpenGL.EGL.EXT.image_dma_buf_import import * +from OpenGL.EGL.KHR.image import * +from OpenGL.EGL.VERSION.EGL_1_0 import * +from OpenGL.EGL.VERSION.EGL_1_2 import * +from OpenGL.EGL.VERSION.EGL_1_3 import * + +from OpenGL.GLES2.OES.EGL_image import * +from OpenGL.GLES2.OES.EGL_image_external import * +from OpenGL.GLES2.VERSION.GLES2_2_0 import * +from OpenGL.GLES3.VERSION.GLES3_3_0 import * + +from OpenGL.GL import shaders + +from gl_helpers import * + +# libcamera format string -> DRM fourcc +FMT_MAP = { + 'RGB888': 'RG24', + 'XRGB8888': 'XR24', + 'ARGB8888': 'AR24', + 'YUYV': 'YUYV', +} + + +class EglState: + def __init__(self): + self.create_display() + self.choose_config() + self.create_context() + self.check_extensions() + + def create_display(self): + xdpy = getEGLNativeDisplay() + dpy = eglGetDisplay(xdpy) + self.display = dpy + + def choose_config(self): + dpy = self.display + + major, minor = EGLint(), EGLint() + + b = eglInitialize(dpy, major, minor) + assert(b) + + print('EGL {} {}'.format( + eglQueryString(dpy, EGL_VENDOR).decode(), + eglQueryString(dpy, EGL_VERSION).decode())) + + check_egl_extensions(dpy, ['EGL_EXT_image_dma_buf_import']) + + b = eglBindAPI(EGL_OPENGL_ES_API) + assert(b) + + def print_config(dpy, cfg): + + def getconf(a): + value = ctypes.c_long() + eglGetConfigAttrib(dpy, cfg, a, value) + return value.value + + print('EGL Config {}: color buf {}/{}/{}/{} = {}, depth {}, stencil {}, native visualid {}, native visualtype {}'.format( + getconf(EGL_CONFIG_ID), + getconf(EGL_ALPHA_SIZE), + getconf(EGL_RED_SIZE), + getconf(EGL_GREEN_SIZE), + getconf(EGL_BLUE_SIZE), + getconf(EGL_BUFFER_SIZE), + getconf(EGL_DEPTH_SIZE), + getconf(EGL_STENCIL_SIZE), + getconf(EGL_NATIVE_VISUAL_ID), + getconf(EGL_NATIVE_VISUAL_TYPE))) + + if False: + num_configs = ctypes.c_long() + eglGetConfigs(dpy, None, 0, num_configs) + print('{} configs'.format(num_configs.value)) + + configs = (EGLConfig * num_configs.value)() + eglGetConfigs(dpy, configs, num_configs.value, num_configs) + for config_id in configs: + print_config(dpy, config_id) + + config_attribs = [ + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 0, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_NONE, + ] + + n = EGLint() + configs = (EGLConfig * 1)() + b = eglChooseConfig(dpy, config_attribs, configs, 1, n) + assert(b and n.value == 1) + config = configs[0] + + print('Chosen Config:') + print_config(dpy, config) + + self.config = config + + def create_context(self): + dpy = self.display + + context_attribs = [ + EGL_CONTEXT_CLIENT_VERSION, 2, + EGL_NONE, + ] + + context = eglCreateContext(dpy, self.config, EGL_NO_CONTEXT, context_attribs) + assert(context) + + b = eglMakeCurrent(dpy, EGL_NO_SURFACE, EGL_NO_SURFACE, context) + assert(b) + + self.context = context + + def check_extensions(self): + check_gl_extensions(['GL_OES_EGL_image']) + + assert(eglCreateImageKHR) + assert(eglDestroyImageKHR) + assert(glEGLImageTargetTexture2DOES) + + +class QtRenderer: + def __init__(self, state): + self.state = state + + def setup(self): + self.app = QtWidgets.QApplication([]) + + window = MainWindow(self.state) + window.setAttribute(QtCore.Qt.WA_ShowWithoutActivating) + window.show() + + self.window = window + + def run(self): + camnotif = QtCore.QSocketNotifier(self.state['cm'].efd, QtCore.QSocketNotifier.Read) + camnotif.activated.connect(lambda x: self.readcam()) + + keynotif = QtCore.QSocketNotifier(sys.stdin.fileno(), QtCore.QSocketNotifier.Read) + keynotif.activated.connect(lambda x: self.readkey()) + + print('Capturing...') + + self.app.exec() + + print('Exiting...') + + def readcam(self): + running = self.state['event_handler'](self.state) + + if not running: + self.app.quit() + + def readkey(self): + sys.stdin.readline() + self.app.quit() + + def request_handler(self, ctx, req): + self.window.handle_request(ctx, req) + + def cleanup(self): + self.window.close() + + +class MainWindow(QtWidgets.QWidget): + def __init__(self, state): + super().__init__() + + self.setAttribute(Qt.WA_PaintOnScreen) + self.setAttribute(Qt.WA_NativeWindow) + + self.state = state + + self.textures = {} + self.reqqueue = {} + self.current = {} + + for ctx in self.state['contexts']: + + self.reqqueue[ctx['idx']] = [] + self.current[ctx['idx']] = [] + + for stream in ctx['streams']: + fmt = stream.configuration.pixel_format + size = stream.configuration.size + + if fmt not in FMT_MAP: + raise Exception('Unsupported pixel format: ' + str(fmt)) + + self.textures[stream] = None + + num_tiles = len(self.textures) + self.num_columns = math.ceil(math.sqrt(num_tiles)) + self.num_rows = math.ceil(num_tiles / self.num_columns) + + self.egl = EglState() + + self.surface = None + + def paintEngine(self): + return None + + def create_surface(self): + native_surface = c_void_p(self.winId().__int__()) + surface = eglCreateWindowSurface(self.egl.display, self.egl.config, + native_surface, None) + + b = eglMakeCurrent(self.egl.display, self.surface, self.surface, self.egl.context) + assert(b) + + self.surface = surface + + def init_gl(self): + self.create_surface() + + vertShaderSrc = ''' + attribute vec2 aPosition; + varying vec2 texcoord; + + void main() + { + gl_Position = vec4(aPosition * 2.0 - 1.0, 0.0, 1.0); + texcoord.x = aPosition.x; + texcoord.y = 1.0 - aPosition.y; + } + ''' + fragShaderSrc = ''' + #extension GL_OES_EGL_image_external : enable + precision mediump float; + varying vec2 texcoord; + uniform samplerExternalOES texture; + + void main() + { + gl_FragColor = texture2D(texture, texcoord); + } + ''' + + program = shaders.compileProgram( + shaders.compileShader(vertShaderSrc, GL_VERTEX_SHADER), + shaders.compileShader(fragShaderSrc, GL_FRAGMENT_SHADER) + ) + + glUseProgram(program) + + glClearColor(0.5, 0.8, 0.7, 1.0) + + vertPositions = [ + 0.0, 0.0, + 1.0, 0.0, + 1.0, 1.0, + 0.0, 1.0 + ] + + inputAttrib = glGetAttribLocation(program, 'aPosition') + glVertexAttribPointer(inputAttrib, 2, GL_FLOAT, GL_FALSE, 0, vertPositions) + glEnableVertexAttribArray(inputAttrib) + + def create_texture(self, stream, fb): + cfg = stream.configuration + fmt = cfg.pixel_format + fmt = str_to_fourcc(FMT_MAP[fmt]) + w, h = cfg.size + + attribs = [ + EGL_WIDTH, w, + EGL_HEIGHT, h, + EGL_LINUX_DRM_FOURCC_EXT, fmt, + EGL_DMA_BUF_PLANE0_FD_EXT, fb.fd(0), + EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0, + EGL_DMA_BUF_PLANE0_PITCH_EXT, cfg.stride, + EGL_NONE, + ] + + image = eglCreateImageKHR(self.egl.display, + EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + None, + attribs) + assert(image) + + textures = glGenTextures(1) + glBindTexture(GL_TEXTURE_EXTERNAL_OES, textures) + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR) + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE) + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE) + glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, image) + + return textures + + def resizeEvent(self, event): + size = event.size() + + print('Resize', size) + + super().resizeEvent(event) + + if self.surface is None: + return + + glViewport(0, 0, size.width() // 2, size.height()) + + def paintEvent(self, event): + if self.surface is None: + self.init_gl() + + for ctx_idx, queue in self.reqqueue.items(): + if len(queue) == 0: + continue + + ctx = next(ctx for ctx in self.state['contexts'] if ctx['idx'] == ctx_idx) + + if self.current[ctx_idx]: + old = self.current[ctx_idx] + self.current[ctx_idx] = None + self.state['request_prcessed'](ctx, old) + + next_req = queue.pop(0) + self.current[ctx_idx] = next_req + + stream, fb = next(iter(next_req.buffers.items())) + + self.textures[stream] = self.create_texture(stream, fb) + + self.paint_gl() + + def paint_gl(self): + b = eglMakeCurrent(self.egl.display, self.surface, self.surface, self.egl.context) + assert(b) + + glClear(GL_COLOR_BUFFER_BIT) + + size = self.size() + + for idx, ctx in enumerate(self.state['contexts']): + for stream in ctx['streams']: + if self.textures[stream] is None: + continue + + w = size.width() // self.num_columns + h = size.height() // self.num_rows + + x = idx % self.num_columns + y = idx // self.num_columns + + x *= w + y *= h + + glViewport(x, y, w, h) + + glBindTexture(GL_TEXTURE_EXTERNAL_OES, self.textures[stream]) + glDrawArrays(GL_TRIANGLE_FAN, 0, 4) + + b = eglSwapBuffers(self.egl.display, self.surface) + assert(b) + + def handle_request(self, ctx, req): + self.reqqueue[ctx['idx']].append(req) + self.update() diff --git a/src/py/cam/gl_helpers.py b/src/py/cam/gl_helpers.py new file mode 100644 index 00000000..ac5e6889 --- /dev/null +++ b/src/py/cam/gl_helpers.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2022, Tomi Valkeinen <tomi.valkeinen@ideasonboard.com> + +from OpenGL.EGL.VERSION.EGL_1_0 import EGLNativeDisplayType, eglGetProcAddress, eglQueryString, EGL_EXTENSIONS + +from OpenGL.raw.GLES2 import _types as _cs +from OpenGL.GLES2.VERSION.GLES2_2_0 import * +from OpenGL.GLES3.VERSION.GLES3_3_0 import * +from OpenGL import GL as gl + +from ctypes import c_int, c_char_p, c_void_p, cdll, POINTER, util, \ + pointer, CFUNCTYPE, c_bool + + +def getEGLNativeDisplay(): + _x11lib = cdll.LoadLibrary(util.find_library('X11')) + XOpenDisplay = _x11lib.XOpenDisplay + XOpenDisplay.argtypes = [c_char_p] + XOpenDisplay.restype = POINTER(EGLNativeDisplayType) + + return XOpenDisplay(None) + + +# Hack. PyOpenGL doesn't seem to manage to find glEGLImageTargetTexture2DOES. +def getglEGLImageTargetTexture2DOES(): + funcptr = eglGetProcAddress('glEGLImageTargetTexture2DOES') + prototype = CFUNCTYPE(None, _cs.GLenum, _cs.GLeglImageOES) + return prototype(funcptr) + + +glEGLImageTargetTexture2DOES = getglEGLImageTargetTexture2DOES() + +# \todo This can be dropped when we have proper PixelFormat bindings +def str_to_fourcc(str): + assert(len(str) == 4) + fourcc = 0 + for i, v in enumerate([ord(c) for c in str]): + fourcc |= v << (i * 8) + return fourcc + + +def get_gl_extensions(): + n = GLint() + glGetIntegerv(GL_NUM_EXTENSIONS, n) + gl_extensions = [] + for i in range(n.value): + gl_extensions.append(gl.glGetStringi(GL_EXTENSIONS, i).decode()) + return gl_extensions + + +def check_gl_extensions(required_extensions): + extensions = get_gl_extensions() + + if False: + print('GL EXTENSIONS: ', ' '.join(extensions)) + + for ext in required_extensions: + if ext not in extensions: + raise Exception(ext + ' missing') + + +def get_egl_extensions(egl_display): + return eglQueryString(egl_display, EGL_EXTENSIONS).decode().split(' ') + + +def check_egl_extensions(egl_display, required_extensions): + extensions = get_egl_extensions(egl_display) + + if False: + print('EGL EXTENSIONS: ', ' '.join(extensions)) + + for ext in required_extensions: + if ext not in extensions: + raise Exception(ext + ' missing') |