/* SPDX-License-Identifier: LGPL-2.1-or-later */ /* * Copyright (C) 2019, Google Inc. * * object.cpp - Base object */ #include #include #include #include "libcamera/internal/log.h" #include "libcamera/internal/message.h" #include "libcamera/internal/semaphore.h" #include "libcamera/internal/thread.h" #include "libcamera/internal/utils.h" /** * \file object.h * \brief Base object to support automatic signal disconnection */ namespace libcamera { LOG_DEFINE_CATEGORY(Object) /** * \class Object * \brief Base object to support automatic signal disconnection * * The Object class simplifies signal/slot handling for classes implementing * slots. By inheriting from Object, an object is automatically disconnected * from all connected signals when it gets destroyed. * * Object instances are bound to the thread of their parent, or the thread in * which they're created when they have no parent. When a message is posted to * an object, its handler will run in the object's thread. This allows * implementing easy message passing between threads by inheriting from the * Object class. * * Deleting an object from a thread other than the one the object is bound to is * unsafe, unless the caller ensures that the object isn't processing any * message concurrently. * * Object slots connected to signals will also run in the context of the * object's thread, regardless of whether the signal is emitted in the same or * in another thread. * * \sa Message, Signal, Thread */ /** * \brief Construct an Object instance * \param[in] parent The object parent * * The new Object instance is bound to the thread of its \a parent, or to the * current thread if the \a parent is nullptr. */ Object::Object(Object *parent) : parent_(parent), pendingMessages_(0) { thread_ = parent ? parent->thread() : Thread::current(); if (parent) parent->children_.push_back(this); } /** * \brief Destroy an Object instance * * Deleting an Object automatically disconnects all signals from the Object's * slots. All the Object's children are made orphan, but stay bound to their * current thread. * * Object instances shall be destroyed from the thread they are bound to, * otherwise undefined behaviour may occur. If deletion of an Object needs to * be scheduled from a different thread, deleteLater() shall be used. */ Object::~Object() { /* * Move signals to a private list to avoid concurrent iteration and * deletion of items from Signal::disconnect(). */ std::list signals(std::move(signals_)); for (SignalBase *signal : signals) signal->disconnect(this); if (pendingMessages_) thread()->removeMessages(this); if (parent_) { auto it = std::find(parent_->children_.begin(), parent_->children_.end(), this); ASSERT(it != parent_->children_.end()); parent_->children_.erase(it); } for (auto child : children_) child->parent_ = nullptr; } /** * \brief Schedule deletion of the instance in the thread it belongs to * * This function schedules deletion of the Object when control returns to the * event loop that the object belongs to. This ensures the object is destroyed * from the right context, as required by the libcamera threading model. * * If this function is called before the thread's event loop is started, the * object will be deleted when the event loop starts. * * Deferred deletion can be used to control the destruction context with shared * pointers. An object managed with shared pointers is deleted when the last * reference is destroyed, which makes difficult to ensure through software * design which context the deletion will take place in. With a custom deleter * for the shared pointer using deleteLater(), the deletion can be guaranteed to * happen in the thread the object is bound to. * * \code{.cpp} * std::shared_ptr createObject() * { * struct Deleter : std::default_delete { * void operator()(MyObject *obj) * { * obj->deleteLater(); * } * }; * * MyObject *obj = new MyObject(); * * return std::shared_ptr(obj, Deleter()); * } * \endcode * * \context This function is \threadsafe. */ void Object::deleteLater() { postMessage(std::make_unique(Message::DeferredDelete)); } /** * \brief Post a message to the object's thread * \param[in] msg The message * * This method posts the message \a msg to the message queue of the object's * thread, to be delivered to the object through the message() method in the * context of its thread. Message ownership is passed to the thread, and the * message will be deleted after being delivered. * * Messages are delivered through the thread's event loop. If the thread is not * running its event loop the message will not be delivered until the event * loop gets started. * * \context This function is \threadsafe. */ void Object::postMessage(std::unique_ptr msg) { thread()->postMessage(std::move(msg), this); } /** * \brief Message handler for the object * \param[in] msg The message * * This virtual method receives messages for the object. It is called in the * context of the object's thread, and can be overridden to process custom * messages. The parent Object::message() method shall be called for any * message not handled by the override method. * * The message \a msg is valid only for the duration of the call, no reference * to it shall be kept after this method returns. */ void Object::message(Message *msg) { switch (msg->type()) { case Message::InvokeMessage: { InvokeMessage *iMsg = static_cast(msg); Semaphore *semaphore = iMsg->semaphore(); iMsg->invoke(); if (semaphore) semaphore->release(); break; } case Message::DeferredDelete: delete this; break; default: break; } } /** * \fn R Object::invokeMethod() * \brief Invoke a method asynchronously on an Object instance * \param[in] func The object method to invoke * \param[in] type Connection type for method invocation * \param[in] args The method arguments * * This method invokes the member method \a func with arguments \a args, based * on the connection \a type. Depending on the type, the method will be called * synchronously in the same thread or asynchronously in the object's thread. * * Arguments \a args passed by value or reference are copied, while pointers * are passed untouched. The caller shall ensure that any pointer argument * remains valid until the method is invoked. * * \context This function is \threadsafe. * * \return For connection types ConnectionTypeDirect and * ConnectionTypeBlocking, return the return value of the invoked method. For * connection type ConnectionTypeQueued, return a default-constructed R value. */ /** * \fn Object::thread() * \brief Retrieve the thread the object is bound to * \context This function is \threadsafe. * \return The thread the object is bound to */ /** * \brief Move the object and all its children to a different thread * \param[in] thread The target thread * * This method moves the object and all its children from the current thread to * the new \a thread. * * Before the object is moved, a Message::ThreadMoveMessage message is sent to * it. The message() method can be reimplement in derived classes to be notified * of the upcoming thread move and perform any required processing. * * Moving an object that has a parent is not allowed, and causes undefined * behaviour. * * \context This function is thread-bound. */ void Object::moveToThread(Thread *thread) { ASSERT(Thread::current() == thread_); if (thread_ == thread) return; if (parent_) { LOG(Object, Error) << "Moving object to thread with a parent is not permitted"; return; } notifyThreadMove(); thread->moveObject(this); } void Object::notifyThreadMove() { Message msg(Message::ThreadMoveMessage); message(&msg); for (auto child : children_) child->notifyThreadMove(); } /** * \fn Object::parent() * \brief Retrieve the object's parent * \return The object's parent */ void Object::connect(SignalBase *signal) { signals_.push_back(signal); } void Object::disconnect(SignalBase *signal) { for (auto iter = signals_.begin(); iter != signals_.end(); ) { if (*iter == signal) iter = signals_.erase(iter); else iter++; } } } /* namespace libcamera */ #n160'>160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404

# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (C) 2019, Raspberry Pi Ltd
#
# camera tuning tool for CCM (colour correction matrix)

from ctt_image_load import *
from ctt_awb import get_alsc_patches
import colors
from scipy.optimize import minimize
from ctt_visualise import visualise_macbeth_chart
import numpy as np
"""
takes 8-bit macbeth chart values, degammas and returns 16 bit
"""

'''
This program has many options from which to derive the color matrix from.
The first is average. This minimises the average delta E across all patches of
the macbeth chart. Testing across all cameras yeilded this as the most color
accurate and vivid. Other options are avalible however.
Maximum minimises the maximum Delta E of the patches. It iterates through till
a minimum maximum is found (so that there is
not one patch that deviates wildly.)
This yields generally good results but overall the colors are less accurate
Have a fiddle with maximum and see what you think.
The final option allows you to select the patches for which to average across.
This means that you can bias certain patches, for instance if you want the
reds to be more accurate.
'''

matrix_selection_types = ["average", "maximum", "patches"]
typenum = 0  # select from array above, 0 = average, 1 = maximum, 2 = patches
test_patches = [1, 2, 5, 8, 9, 12, 14]

'''
Enter patches to test for. Can also be entered twice if you
would like twice as much bias on one patch.
'''


def degamma(x):
    x = x / ((2 ** 8) - 1)  # takes 255 and scales it down to one
    x = np.where(x < 0.04045, x / 12.92, ((x + 0.055) / 1.055) ** 2.4)
    x = x * ((2 ** 16) - 1)  # takes one and scales up to 65535, 16 bit color
    return x


def gamma(x):
    # Take 3 long array of color values and gamma them
    return [((colour / 255) ** (1 / 2.4) * 1.055 - 0.055) * 255 for colour in x]


"""
FInds colour correction matrices for list of images
"""


def ccm(Cam, cal_cr_list, cal_cb_list, grid_size):
    global matrix_selection_types, typenum
    imgs = Cam.imgs
    """
    standard macbeth chart colour values
    """
    m_rgb = np.array([  # these are in RGB
        [116, 81, 67],    # dark skin
        [199, 147, 129],  # light skin
        [91, 122, 156],   # blue sky
        [90, 108, 64],    # foliage
        [130, 128, 176],  # blue flower
        [92, 190, 172],   # bluish green
        [224, 124, 47],   # orange
        [68, 91, 170],    # purplish blue
        [198, 82, 97],    # moderate red
        [94, 58, 106],    # purple
        [159, 189, 63],   # yellow green
        [230, 162, 39],   # orange yellow
        [35, 63, 147],    # blue
        [67, 149, 74],    # green
        [180, 49, 57],    # red
        [238, 198, 20],   # yellow
        [193, 84, 151],   # magenta
        [0, 136, 170],    # cyan (goes out of gamut)
        [245, 245, 243],  # white 9.5
        [200, 202, 202],  # neutral 8
        [161, 163, 163],  # neutral 6.5
        [121, 121, 122],  # neutral 5
        [82, 84, 86],     # neutral 3.5
        [49, 49, 51]      # black 2
    ])
    """
    convert reference colours from srgb to rgb
    """
    m_srgb = degamma(m_rgb)  # now in 16 bit color.

    # Produce array of LAB values for ideal color chart
    m_lab = [colors.RGB_to_LAB(color / 256) for color in m_srgb]

    """
    reorder reference values to match how patches are ordered
    """
    m_srgb = np.array([m_srgb[i::6] for i in range(6)]).reshape((24, 3))
    m_lab = np.array([m_lab[i::6] for i in range(6)]).reshape((24, 3))
    m_rgb = np.array([m_rgb[i::6] for i in range(6)]).reshape((24, 3))
    """
    reformat alsc correction tables or set colour_cals to None if alsc is
    deactivated
    """
    if cal_cr_list is None:
        colour_cals = None
    else:
        colour_cals = {}
        for cr, cb in zip(cal_cr_list, cal_cb_list):
            cr_tab = cr['table']
            cb_tab = cb['table']
            """
            normalise tables so min value is 1
            """
            cr_tab = cr_tab / np.min(cr_tab)
            cb_tab = cb_tab / np.min(cb_tab)
            colour_cals[cr['ct']] = [cr_tab, cb_tab]

    """
    for each image, perform awb and alsc corrections.
    Then calculate the colour correction matrix for that image, recording the
    ccm and the colour tempertaure.
    """
    ccm_tab = {}
    for Img in imgs:
        Cam.log += '\nProcessing image: ' + Img.name
        """
        get macbeth patches with alsc applied if alsc enabled.
        Note: if alsc is disabled then colour_cals will be set to None and no
        the function will simply return the macbeth patches
        """
        r, b, g = get_alsc_patches(Img, colour_cals, grey=False, grid_size=grid_size)
        """
        do awb
        Note: awb is done by measuring the macbeth chart in the image, rather
        than from the awb calibration. This is done so the awb will be perfect
        and the ccm matrices will be more accurate.
        """
        r_greys, b_greys, g_greys = r[3::4], b[3::4], g[3::4]
        r_g = np.mean(r_greys / g_greys)
        b_g = np.mean(b_greys / g_greys)
        r = r / r_g
        b = b / b_g
        """
        normalise brightness wrt reference macbeth colours and then average
        each channel for each patch
        """
        gain = np.mean(m_srgb) / np.mean((r, g, b))
        Cam.log += '\nGain with respect to standard colours: {:.3f}'.format(gain)
        r = np.mean(gain * r, axis=1)
        b = np.mean(gain * b, axis=1)
        g = np.mean(gain * g, axis=1)
        """
        calculate ccm matrix
        """
        # ==== All of below should in sRGB ===##
        sumde = 0
        ccm = do_ccm(r, g, b, m_srgb)
        # This is the initial guess that our optimisation code works with.
        original_ccm = ccm
        r1 = ccm[0]
        r2 = ccm[1]
        g1 = ccm[3]
        g2 = ccm[4]
        b1 = ccm[6]
        b2 = ccm[7]
        '''
        COLOR MATRIX LOOKS AS BELOW
        R1 R2 R3   Rval     Outr
        G1 G2 G3  *  Gval  =  G
        B1 B2 B3   Bval     B
        Will be optimising 6 elements and working out the third element using 1-r1-r2 = r3
        '''

        x0 = [r1, r2, g1, g2, b1, b2]
        '''
        We use our old CCM as the initial guess for the program to find the
        optimised matrix
        '''
        result = minimize(guess, x0, args=(r, g, b, m_lab), tol=0.01)
        '''
        This produces a color matrix which has the lowest delta E possible,
        based off the input data. Note it is impossible for this to reach
        zero since the input data is imperfect
        '''

        Cam.log += ("\n \n Optimised Matrix Below: \n \n")
        [r1, r2, g1, g2, b1, b2] = result.x
        # The new, optimised color correction matrix values
        optimised_ccm = [r1, r2, (1 - r1 - r2), g1, g2, (1 - g1 - g2), b1, b2, (1 - b1 - b2)]

        # This is the optimised Color Matrix (preserving greys by summing rows up to 1)
        Cam.log += str(optimised_ccm)
        Cam.log += "\n Old Color Correction Matrix Below \n"
        Cam.log += str(ccm)

        formatted_ccm = np.array(original_ccm).reshape((3, 3))

        '''
        below is a whole load of code that then applies the latest color
        matrix, and returns LAB values for color. This can then be used
        to calculate the final delta E
        '''
        optimised_ccm_rgb = []  # Original Color Corrected Matrix RGB / LAB
        optimised_ccm_lab = []

        formatted_optimised_ccm = np.array(optimised_ccm).reshape((3, 3))
        after_gamma_rgb = []
        after_gamma_lab = []

        for RGB in zip(r, g, b):
            ccm_applied_rgb = np.dot(formatted_ccm, (np.array(RGB) / 256))
            optimised_ccm_rgb.append(gamma(ccm_applied_rgb))
            optimised_ccm_lab.append(colors.RGB_to_LAB(ccm_applied_rgb))

            optimised_ccm_applied_rgb = np.dot(formatted_optimised_ccm, np.array(RGB) / 256)
            after_gamma_rgb.append(gamma(optimised_ccm_applied_rgb))
            after_gamma_lab.append(colors.RGB_to_LAB(optimised_ccm_applied_rgb))
        '''
        Gamma After RGB / LAB - not used in calculations, only used for visualisation
        We now want to spit out some data that shows
        how the optimisation has improved the color matrices
        '''
        Cam.log += "Here are the Improvements"

        # CALCULATE WORST CASE delta e
        old_worst_delta_e = 0
        before_average = transform_and_evaluate(formatted_ccm, r, g, b, m_lab)
        new_worst_delta_e = 0
        after_average = transform_and_evaluate(formatted_optimised_ccm, r, g, b, m_lab)
        for i in range(24):
            old_delta_e = deltae(optimised_ccm_lab[i], m_lab[i])  # Current Old Delta E
            new_delta_e = deltae(after_gamma_lab[i], m_lab[i])  # Current New Delta E
            if old_delta_e > old_worst_delta_e:
                old_worst_delta_e = old_delta_e
            if new_delta_e > new_worst_delta_e:
                new_worst_delta_e = new_delta_e

        Cam.log += "Before color correction matrix was optimised, we got an average delta E of " + str(before_average) + " and a maximum delta E of " + str(old_worst_delta_e)
        Cam.log += "After color correction matrix was optimised, we got an average delta E of " + str(after_average) + " and a maximum delta E of " + str(new_worst_delta_e)

        visualise_macbeth_chart(m_rgb, optimised_ccm_rgb, after_gamma_rgb, str(Img.col) + str(matrix_selection_types[typenum]))
        '''
        The program will also save some visualisations of improvements.
        Very pretty to look at. Top rectangle is ideal, Left square is
        before optimisation, right square is after.
        '''

        """
        if a ccm has already been calculated for that temperature then don't
        overwrite but save both. They will then be averaged later on
        """  # Now going to use optimised color matrix, optimised_ccm
        if Img.col in ccm_tab.keys():
            ccm_tab[Img.col].append(optimised_ccm)
        else:
            ccm_tab[Img.col] = [optimised_ccm]
        Cam.log += '\n'

    Cam.log += '\nFinished processing images'
    """
    average any ccms that share a colour temperature
    """
    for k, v in ccm_tab.items():
        tab = np.mean(v, axis=0)
        tab = np.where((10000 * tab) % 1 <= 0.05, tab + 0.00001, tab)
        tab = np.where((10000 * tab) % 1 >= 0.95, tab - 0.00001, tab)
        ccm_tab[k] = list(np.round(tab, 5))
        Cam.log += '\nMatrix calculated for colour temperature of {} K'.format(k)

    """
    return all ccms with respective colour temperature in the correct format,
    sorted by their colour temperature
    """
    sorted_ccms = sorted(ccm_tab.items(), key=lambda kv: kv[0])
    ccms = []
    for i in sorted_ccms:
        ccms.append({
            'ct': i[0],
            'ccm': i[1]
        })
    return ccms


def guess(x0, r, g, b, m_lab):       # provides a method of numerical feedback for the optimisation code
    [r1, r2, g1, g2, b1, b2] = x0
    ccm = np.array([r1, r2, (1 - r1 - r2),
                    g1, g2, (1 - g1 - g2),
                    b1, b2, (1 - b1 - b2)]).reshape((3, 3))  # format the matrix correctly
    return transform_and_evaluate(ccm, r, g, b, m_lab)


def transform_and_evaluate(ccm, r, g, b, m_lab):  # Transforms colors to LAB and applies the correction matrix
    # create list of matrix changed colors
    realrgb = []
    for RGB in zip(r, g, b):
        rgb_post_ccm = np.dot(ccm, np.array(RGB) / 256)  # This is RGB values after the color correction matrix has been applied
        realrgb.append(colors.RGB_to_LAB(rgb_post_ccm))
    # now compare that with m_lab and return numeric result, averaged for each patch
    return (sumde(realrgb, m_lab) / 24)  # returns an average result of delta E


def sumde(listA, listB):
    global typenum, test_patches
    sumde = 0
    maxde = 0
    patchde = []  # Create array of the delta E values for each patch. useful for optimisation of certain patches
    for listA_item, listB_item in zip(listA, listB):
        if maxde < (deltae(listA_item, listB_item)):
            maxde = deltae(listA_item, listB_item)
        patchde.append(deltae(listA_item, listB_item))
        sumde += deltae(listA_item, listB_item)
    '''
    The different options specified at the start allow for
    the maximum to be returned, average or specific patches
    '''
    if typenum == 0:
        return sumde
    if typenum == 1:
        return maxde
    if typenum == 2:
        output = sum([patchde[test_patch] for test_patch in test_patches])
        # Selects only certain patches and returns the output for them
        return output


"""
calculates the ccm for an individual image.
ccms are calculated in rgb space, and are fit by hand. Although it is a 3x3
matrix, each row must add up to 1 in order to conserve greyness, simplifying
calculation.
The initial CCM is calculated in RGB, and then optimised in LAB color space
This simplifies the initial calculation but then gets us the accuracy of
using LAB color space.
"""


def do_ccm(r, g, b, m_srgb):
    rb = r-b
    gb = g-b
    rb_2s = (rb * rb)
    rb_gbs = (rb * gb)
    gb_2s = (gb * gb)

    r_rbs = rb * (m_srgb[..., 0] - b)
    r_gbs = gb * (m_srgb[..., 0] - b)
    g_rbs = rb * (m_srgb[..., 1] - b)
    g_gbs = gb * (m_srgb[..., 1] - b)
    b_rbs = rb * (m_srgb[..., 2] - b)
    b_gbs = gb * (m_srgb[..., 2] - b)

    """
    Obtain least squares fit
    """
    rb_2 = np.sum(rb_2s)
    gb_2 = np.sum(gb_2s)
    rb_gb = np.sum(rb_gbs)
    r_rb = np.sum(r_rbs)
    r_gb = np.sum(r_gbs)
    g_rb = np.sum(g_rbs)
    g_gb = np.sum(g_gbs)
    b_rb = np.sum(b_rbs)
    b_gb = np.sum(b_gbs)

    det = rb_2 * gb_2 - rb_gb * rb_gb

    """
    Raise error if matrix is singular...
    This shouldn't really happen with real data but if it does just take new
    pictures and try again, not much else to be done unfortunately...
    """
    if det < 0.001:
        raise ArithmeticError

    r_a = (gb_2 * r_rb - rb_gb * r_gb) / det
    r_b = (rb_2 * r_gb - rb_gb * r_rb) / det
    """
    Last row can be calculated by knowing the sum must be 1
    """
    r_c = 1 - r_a - r_b

    g_a = (gb_2 * g_rb - rb_gb * g_gb) / det
    g_b = (rb_2 * g_gb - rb_gb * g_rb) / det
    g_c = 1 - g_a - g_b

    b_a = (gb_2 * b_rb - rb_gb * b_gb) / det
    b_b = (rb_2 * b_gb - rb_gb * b_rb) / det
    b_c = 1 - b_a - b_b

    """
    format ccm
    """
    ccm = [r_a, r_b, r_c, g_a, g_b, g_c, b_a, b_b, b_c]

    return ccm


def deltae(colorA, colorB):
    return ((colorA[0] - colorB[0]) ** 2 + (colorA[1] - colorB[1]) ** 2 + (colorA[2] - colorB[2]) ** 2) ** 0.5
    # return ((colorA[1]-colorB[1]) *  * 2 + (colorA[2]-colorB[2]) *  * 2) *  * 0.5
    # UNCOMMENT IF YOU WANT TO NEGLECT LUMINANCE FROM CALCULATION OF DELTA E