summaryrefslogtreecommitdiff
path: root/utils/raspberrypi/ctt/ctt_ccm.py
blob: cebecfc248a6a143f9612edaaa955ceab7fbdf06 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (C) 2019, Raspberry Pi (Trading) Limited
#
# ctt_ccm.py - camera tuning tool for CCM (colour correction matrix)

from ctt_image_load import *
from ctt_awb import get_alsc_patches


"""
takes 8-bit macbeth chart values, degammas and returns 16 bit
"""
def degamma(x):
    x = x / ((2**8)-1)
    x = np.where(x < 0.04045, x/12.92, ((x+0.055)/1.055)**2.4)
    x = x * ((2**16)-1)
    return x


"""
FInds colour correction matrices for list of images
"""
def ccm(Cam, cal_cr_list, cal_cb_list):
    imgs = Cam.imgs
    """
    standard macbeth chart colour values
    """
    m_rgb = np.array([  # these are in sRGB
        [116, 81, 67],    # dark skin
        [199, 147, 129],  # light skin
        [91, 122, 156],   # blue sky
        [90, 108, 64],    # foliage
        [130, 128, 176],  # blue flower
        [92, 190, 172],   # bluish green
        [224, 124, 47],   # orange
        [68, 91, 170],     # purplish blue
        [198, 82, 97],    # moderate red
        [94, 58, 106],    # purple
        [159, 189, 63],   # yellow green
        [230, 162, 39],   # orange yellow
        [35, 63, 147],    # blue
        [67, 149, 74],    # green
        [180, 49, 57],    # red
        [238, 198, 20],   # yellow
        [193, 84, 151],   # magenta
        [0, 136, 170],    # cyan (goes out of gamut)
        [245, 245, 243],  # white 9.5
        [200, 202, 202],  # neutral 8
        [161, 163, 163],  # neutral 6.5
        [121, 121, 122],  # neutral 5
        [82, 84, 86],     # neutral 3.5
        [49, 49, 51]      # black 2
    ])

    """
    convert reference colours from srgb to rgb
    """
    m_srgb = degamma(m_rgb)
    """
    reorder reference values to match how patches are ordered
    """
    m_srgb = np.array([m_srgb[i::6] for i in range(6)]).reshape((24, 3))

    """
    reformat alsc correction tables or set colour_cals to None if alsc is
    deactivated
    """
    if cal_cr_list is None:
        colour_cals = None
    else:
        colour_cals = {}
        for cr, cb in zip(cal_cr_list, cal_cb_list):
            cr_tab = cr['table']
            cb_tab = cb['table']
            """
            normalise tables so min value is 1
            """
            cr_tab = cr_tab/np.min(cr_tab)
            cb_tab = cb_tab/np.min(cb_tab)
            colour_cals[cr['ct']] = [cr_tab, cb_tab]

    """
    for each image, perform awb and alsc corrections.
    Then calculate the colour correction matrix for that image, recording the
    ccm and the colour tempertaure.
    """
    ccm_tab = {}
    for Img in imgs:
        Cam.log += '\nProcessing image: ' + Img.name
        """
        get macbeth patches with alsc applied if alsc enabled.
        Note: if alsc is disabled then colour_cals will be set to None and no
        the function will simply return the macbeth patches
        """
        r, b, g = get_alsc_patches(Img, colour_cals, grey=False)
        """
        do awb
        Note: awb is done by measuring the macbeth chart in the image, rather
        than from the awb calibration. This is done so the awb will be perfect
        and the ccm matrices will be more accurate.
        """
        r_greys, b_greys, g_greys = r[3::4], b[3::4], g[3::4]
        r_g = np.mean(r_greys/g_greys)
        b_g = np.mean(b_greys/g_greys)
        r = r / r_g
        b = b / b_g

        """
        normalise brightness wrt reference macbeth colours and then average
        each channel for each patch
        """
        gain = np.mean(m_srgb)/np.mean((r, g, b))
        Cam.log += '\nGain with respect to standard colours: {:.3f}'.format(gain)
        r = np.mean(gain*r, axis=1)
        b = np.mean(gain*b, axis=1)
        g = np.mean(gain*g, axis=1)

        """
        calculate ccm matrix
        """
        ccm = do_ccm(r, g, b, m_srgb)

        """
        if a ccm has already been calculated for that temperature then don't
        overwrite but save both. They will then be averaged later on
        """
        if Img.col in ccm_tab.keys():
            ccm_tab[Img.col].append(ccm)
        else:
            ccm_tab[Img.col] = [ccm]
        Cam.log += '\n'

    Cam.log += '\nFinished processing images'
    """
    average any ccms that share a colour temperature
    """
    for k, v in ccm_tab.items():
        tab = np.mean(v, axis=0)
        tab = np.where((10000*tab) % 1 <= 0.05, tab+0.00001, tab)
        tab = np.where((10000*tab) % 1 >= 0.95, tab-0.00001, tab)
        ccm_tab[k] = list(np.round(tab, 5))
        Cam.log += '\nMatrix calculated for colour temperature of {} K'.format(k)

    """
    return all ccms with respective colour temperature in the correct format,
    sorted by their colour temperature
    """
    sorted_ccms = sorted(ccm_tab.items(), key=lambda kv: kv[0])
    ccms = []
    for i in sorted_ccms:
        ccms.append({
            'ct': i[0],
            'ccm': i[1]
        })
    return ccms


"""
calculates the ccm for an individual image.
ccms are calculate in rgb space, and are fit by hand. Although it is a 3x3
matrix, each row must add up to 1 in order to conserve greyness, simplifying
calculation.
Should you want to fit them in another space (e.g. LAB) we wish you the best of
luck and send us the code when you are done! :-)
"""
def do_ccm(r, g, b, m_srgb):
    rb = r-b
    gb = g-b
    rb_2s = (rb*rb)
    rb_gbs = (rb*gb)
    gb_2s = (gb*gb)

    r_rbs = rb * (m_srgb[..., 0] - b)
    r_gbs = gb * (m_srgb[..., 0] - b)
    g_rbs = rb * (m_srgb[..., 1] - b)
    g_gbs = gb * (m_srgb[..., 1] - b)
    b_rbs = rb * (m_srgb[..., 2] - b)
    b_gbs = gb * (m_srgb[..., 2] - b)

    """
    Obtain least squares fit
    """
    rb_2 = np.sum(rb_2s)
    gb_2 = np.sum(gb_2s)
    rb_gb = np.sum(rb_gbs)
    r_rb = np.sum(r_rbs)
    r_gb = np.sum(r_gbs)
    g_rb = np.sum(g_rbs)
    g_gb = np.sum(g_gbs)
    b_rb = np.sum(b_rbs)
    b_gb = np.sum(b_gbs)

    det = rb_2*gb_2 - rb_gb*rb_gb

    """
    Raise error if matrix is singular...
    This shouldn't really happen with real data but if it does just take new
    pictures and try again, not much else to be done unfortunately...
    """
    if det < 0.001:
        raise ArithmeticError

    r_a = (gb_2*r_rb - rb_gb*r_gb)/det
    r_b = (rb_2*r_gb - rb_gb*r_rb)/det
    """
    Last row can be calculated by knowing the sum must be 1
    """
    r_c = 1 - r_a - r_b

    g_a = (gb_2*g_rb - rb_gb*g_gb)/det
    g_b = (rb_2*g_gb - rb_gb*g_rb)/det
    g_c = 1 - g_a - g_b

    b_a = (gb_2*b_rb - rb_gb*b_gb)/det
    b_b = (rb_2*b_gb - rb_gb*b_rb)/det
    b_c = 1 - b_a - b_b

    """
    format ccm
    """
    ccm = [r_a, r_b, r_c, g_a, g_b, g_c, b_a, b_b, b_c]

    return ccm
t">[3] g2 = ccm[4] b1 = ccm[6] b2 = ccm[7] ''' COLOR MATRIX LOOKS AS BELOW R1 R2 R3 Rval Outr G1 G2 G3 * Gval = G B1 B2 B3 Bval B Will be optimising 6 elements and working out the third element using 1-r1-r2 = r3 ''' x0 = [r1, r2, g1, g2, b1, b2] ''' We use our old CCM as the initial guess for the program to find the optimised matrix ''' result = minimize(guess, x0, args=(r, g, b, m_lab), tol=0.01) ''' This produces a color matrix which has the lowest delta E possible, based off the input data. Note it is impossible for this to reach zero since the input data is imperfect ''' Cam.log += ("\n \n Optimised Matrix Below: \n \n") [r1, r2, g1, g2, b1, b2] = result.x # The new, optimised color correction matrix values optimised_ccm = [r1, r2, (1 - r1 - r2), g1, g2, (1 - g1 - g2), b1, b2, (1 - b1 - b2)] # This is the optimised Color Matrix (preserving greys by summing rows up to 1) Cam.log += str(optimised_ccm) Cam.log += "\n Old Color Correction Matrix Below \n" Cam.log += str(ccm) formatted_ccm = np.array(original_ccm).reshape((3, 3)) ''' below is a whole load of code that then applies the latest color matrix, and returns LAB values for color. This can then be used to calculate the final delta E ''' optimised_ccm_rgb = [] # Original Color Corrected Matrix RGB / LAB optimised_ccm_lab = [] formatted_optimised_ccm = np.array(optimised_ccm).reshape((3, 3)) after_gamma_rgb = [] after_gamma_lab = [] for RGB in zip(r, g, b): ccm_applied_rgb = np.dot(formatted_ccm, (np.array(RGB) / 256)) optimised_ccm_rgb.append(gamma(ccm_applied_rgb)) optimised_ccm_lab.append(colors.RGB_to_LAB(ccm_applied_rgb)) optimised_ccm_applied_rgb = np.dot(formatted_optimised_ccm, np.array(RGB) / 256) after_gamma_rgb.append(gamma(optimised_ccm_applied_rgb)) after_gamma_lab.append(colors.RGB_to_LAB(optimised_ccm_applied_rgb)) ''' Gamma After RGB / LAB - not used in calculations, only used for visualisation We now want to spit out some data that shows how the optimisation has improved the color matrices ''' Cam.log += "Here are the Improvements" # CALCULATE WORST CASE delta e old_worst_delta_e = 0 before_average = transform_and_evaluate(formatted_ccm, r, g, b, m_lab) new_worst_delta_e = 0 after_average = transform_and_evaluate(formatted_optimised_ccm, r, g, b, m_lab) for i in range(24): old_delta_e = deltae(optimised_ccm_lab[i], m_lab[i]) # Current Old Delta E new_delta_e = deltae(after_gamma_lab[i], m_lab[i]) # Current New Delta E if old_delta_e > old_worst_delta_e: old_worst_delta_e = old_delta_e if new_delta_e > new_worst_delta_e: new_worst_delta_e = new_delta_e Cam.log += "Before color correction matrix was optimised, we got an average delta E of " + str(before_average) + " and a maximum delta E of " + str(old_worst_delta_e) Cam.log += "After color correction matrix was optimised, we got an average delta E of " + str(after_average) + " and a maximum delta E of " + str(new_worst_delta_e) visualise_macbeth_chart(m_rgb, optimised_ccm_rgb, after_gamma_rgb, str(Img.col) + str(matrix_selection_types[typenum])) ''' The program will also save some visualisations of improvements. Very pretty to look at. Top rectangle is ideal, Left square is before optimisation, right square is after. ''' """ if a ccm has already been calculated for that temperature then don't overwrite but save both. They will then be averaged later on """ # Now going to use optimised color matrix, optimised_ccm if Img.col in ccm_tab.keys(): ccm_tab[Img.col].append(optimised_ccm) else: ccm_tab[Img.col] = [optimised_ccm] Cam.log += '\n' Cam.log += '\nFinished processing images' """ average any ccms that share a colour temperature """ for k, v in ccm_tab.items(): tab = np.mean(v, axis=0) tab = np.where((10000 * tab) % 1 <= 0.05, tab + 0.00001, tab) tab = np.where((10000 * tab) % 1 >= 0.95, tab - 0.00001, tab) ccm_tab[k] = list(np.round(tab, 5)) Cam.log += '\nMatrix calculated for colour temperature of {} K'.format(k) """ return all ccms with respective colour temperature in the correct format, sorted by their colour temperature """ sorted_ccms = sorted(ccm_tab.items(), key=lambda kv: kv[0]) ccms = [] for i in sorted_ccms: ccms.append({ 'ct': i[0], 'ccm': i[1] }) return ccms def guess(x0, r, g, b, m_lab): # provides a method of numerical feedback for the optimisation code [r1, r2, g1, g2, b1, b2] = x0 ccm = np.array([r1, r2, (1 - r1 - r2), g1, g2, (1 - g1 - g2), b1, b2, (1 - b1 - b2)]).reshape((3, 3)) # format the matrix correctly return transform_and_evaluate(ccm, r, g, b, m_lab) def transform_and_evaluate(ccm, r, g, b, m_lab): # Transforms colors to LAB and applies the correction matrix # create list of matrix changed colors realrgb = [] for RGB in zip(r, g, b): rgb_post_ccm = np.dot(ccm, np.array(RGB) / 256) # This is RGB values after the color correction matrix has been applied realrgb.append(colors.RGB_to_LAB(rgb_post_ccm)) # now compare that with m_lab and return numeric result, averaged for each patch return (sumde(realrgb, m_lab) / 24) # returns an average result of delta E def sumde(listA, listB): global typenum, test_patches sumde = 0 maxde = 0 patchde = [] # Create array of the delta E values for each patch. useful for optimisation of certain patches for listA_item, listB_item in zip(listA, listB): if maxde < (deltae(listA_item, listB_item)): maxde = deltae(listA_item, listB_item) patchde.append(deltae(listA_item, listB_item)) sumde += deltae(listA_item, listB_item) ''' The different options specified at the start allow for the maximum to be returned, average or specific patches ''' if typenum == 0: return sumde if typenum == 1: return maxde if typenum == 2: output = sum([patchde[test_patch] for test_patch in test_patches]) # Selects only certain patches and returns the output for them return output """ calculates the ccm for an individual image. ccms are calculated in rgb space, and are fit by hand. Although it is a 3x3 matrix, each row must add up to 1 in order to conserve greyness, simplifying calculation. The initial CCM is calculated in RGB, and then optimised in LAB color space This simplifies the initial calculation but then gets us the accuracy of using LAB color space. """ def do_ccm(r, g, b, m_srgb): rb = r-b gb = g-b rb_2s = (rb * rb) rb_gbs = (rb * gb) gb_2s = (gb * gb) r_rbs = rb * (m_srgb[..., 0] - b) r_gbs = gb * (m_srgb[..., 0] - b) g_rbs = rb * (m_srgb[..., 1] - b) g_gbs = gb * (m_srgb[..., 1] - b) b_rbs = rb * (m_srgb[..., 2] - b) b_gbs = gb * (m_srgb[..., 2] - b) """ Obtain least squares fit """ rb_2 = np.sum(rb_2s) gb_2 = np.sum(gb_2s) rb_gb = np.sum(rb_gbs) r_rb = np.sum(r_rbs) r_gb = np.sum(r_gbs) g_rb = np.sum(g_rbs) g_gb = np.sum(g_gbs) b_rb = np.sum(b_rbs) b_gb = np.sum(b_gbs) det = rb_2 * gb_2 - rb_gb * rb_gb """ Raise error if matrix is singular... This shouldn't really happen with real data but if it does just take new pictures and try again, not much else to be done unfortunately... """ if det < 0.001: raise ArithmeticError r_a = (gb_2 * r_rb - rb_gb * r_gb) / det r_b = (rb_2 * r_gb - rb_gb * r_rb) / det """ Last row can be calculated by knowing the sum must be 1 """ r_c = 1 - r_a - r_b g_a = (gb_2 * g_rb - rb_gb * g_gb) / det g_b = (rb_2 * g_gb - rb_gb * g_rb) / det g_c = 1 - g_a - g_b b_a = (gb_2 * b_rb - rb_gb * b_gb) / det b_b = (rb_2 * b_gb - rb_gb * b_rb) / det b_c = 1 - b_a - b_b """ format ccm """ ccm = [r_a, r_b, r_c, g_a, g_b, g_c, b_a, b_b, b_c] return ccm def deltae(colorA, colorB): return ((colorA[0] - colorB[0]) ** 2 + (colorA[1] - colorB[1]) ** 2 + (colorA[2] - colorB[2]) ** 2) ** 0.5 # return ((colorA[1]-colorB[1]) * * 2 + (colorA[2]-colorB[2]) * * 2) * * 0.5 # UNCOMMENT IF YOU WANT TO NEGLECT LUMINANCE FROM CALCULATION OF DELTA E