# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (C) 2019, Raspberry Pi Ltd
#
# camera tuning tool for CCM (colour correction matrix)
from ctt_image_load import *
from ctt_awb import get_alsc_patches
import colors
from scipy.optimize import minimize
from ctt_visualise import visualise_macbeth_chart
import numpy as np
"""
takes 8-bit macbeth chart values, degammas and returns 16 bit
"""
'''
This program has many options from which to derive the color matrix from.
The first is average. This minimises the average delta E across all patches of
the macbeth chart. Testing across all cameras yeilded this as the most color
accurate and vivid. Other options are avalible however.
Maximum minimises the maximum Delta E of the patches. It iterates through till
a minimum maximum is found (so that there is
not one patch that deviates wildly.)
This yields generally good results but overall the colors are less accurate
Have a fiddle with maximum and see what you think.
The final option allows you to select the patches for which to average across.
This means that you can bias certain patches, for instance if you want the
reds to be more accurate.
'''
matrix_selection_types = ["average", "maximum", "patches"]
typenum = 0 # select from array above, 0 = average, 1 = maximum, 2 = patches
test_patches = [1, 2, 5, 8, 9, 12, 14]
'''
Enter patches to test for. Can also be entered twice if you
would like twice as much bias on one patch.
'''
def degamma(x):
x = x / ((2 ** 8) - 1) # takes 255 and scales it down to one
x = np.where(x < 0.04045, x / 12.92, ((x + 0.055) / 1.055) ** 2.4)
x = x * ((2 ** 16) - 1) # takes one and scales up to 65535, 16 bit color
return x
def gamma(x):
# Take 3 long array of color values and gamma them
return [((colour / 255) ** (1 / 2.4) * 1.055 - 0.055) * 255 for colour in x]
"""
FInds colour correction matrices for list of images
"""
def ccm(Cam, cal_cr_list, cal_cb_list, grid_size):
global matrix_selection_types, typenum
imgs = Cam.imgs
"""
standard macbeth chart colour values
"""
m_rgb = np.array([ # these are in RGB
[116, 81, 67], # dark skin
[199, 147, 129], # light skin
[91, 122, 156], # blue sky
[90, 108, 64], # foliage
[130, 128, 176], # blue flower
[92, 190, 172], # bluish green
[224, 124, 47], # orange
[68, 91, 170], # purplish blue
[198, 82, 97], # moderate red
[94, 58, 106], # purple
[159, 189, 63], # yellow green
[230, 162, 39], # orange yellow
[35, 63, 147], # blue
[67, 149, 74], # green
[180, 49, 57], # red
[238, 198, 20], # yellow
[193, 84, 151], # magenta
[0, 136, 170], # cyan (goes out of gamut)
[245, 245, 243], # white 9.5
[200, 202, 202], # neutral 8
[161, 163, 163], # neutral 6.5
[121, 121, 122], # neutral 5
[82, 84, 86], # neutral 3.5
[49, 49, 51] # black 2
])
"""
convert reference colours from srgb to rgb
"""
m_srgb = degamma(m_rgb) # now in 16 bit color.
# Produce array of LAB values for ideal color chart
m_lab = [colors.RGB_to_LAB(color / 256) for color in m_srgb]
"""
reorder reference values to match how patches are ordered
"""
m_srgb = np.array([m_srgb[i::6] for i in range(6)]).reshape((24, 3))
m_lab = np.array([m_lab[i::6] for i in range(6)]).reshape((24, 3))
m_rgb = np.array([m_rgb[i::6] for i in range(6)]).reshape((24, 3))
"""
reformat alsc correction tables or set colour_cals to None if alsc is
deactivated
"""
if cal_cr_list is None:
colour_cals = None
else:
colour_cals = {}
for cr, cb in zip(cal_cr_list, cal_cb_list):
cr_tab = cr['table']
cb_tab = cb['table']
"""
normalise tables so min value is 1
"""
cr_tab = cr_tab / np.min(cr_tab)
cb_tab = cb_tab / np.min(cb_tab)
colour_cals[cr['ct']] = [cr_tab, cb_tab]
"""
for each image, perform awb and alsc corrections.
Then calculate the colour correction matrix for that image, recording the
ccm and the colour tempertaure.
"""
ccm_tab = {}
for Img in imgs:
Cam.log += '\nProcessing image: ' + Img.name
"""
get macbeth patches with alsc applied if alsc enabled.
Note: if alsc is disabled then colour_cals will be set to None and no
the function will simply return the macbeth patches
"""
r, b, g = get_alsc_patches(Img, colour_cals, grey=False, grid_size=grid_size)
"""
do awb
Note: awb is done by measuring the macbeth chart in the image, rather
than from the awb calibration. This is done so the awb will be perfect
and the ccm matrices will be more accurate.
"""
r_greys, b_greys, g_greys = r[3::4], b[3::4], g[3::4]
r_g = np.mean(r_greys / g_greys)
b_g = np.mean(b_greys / g_greys)
r = r / r_g
b = b / b_g
"""
normalise brightness wrt reference macbeth colours and then average
each channel for each patch
"""
gain = np.mean(m_srgb) / np.mean((r, g, b))
Cam.log += '\nGain with respect to standard colours: {:.3f}'.format(gain)
r = np.mean(gain * r, axis=1)
b = np.mean(gain * b, axis=1)
g = np.mean(gain * g, axis=1)
"""
calculate ccm matrix
"""
# ==== All of below should in sRGB ===##
sumde = 0
ccm = do_ccm(r, g, b, m_srgb)
# This is the initial guess that our optimisation code works with.
original_ccm = ccm
r1 = ccm[0]
r2 = ccm[1]
g1 = ccm[3]
g2 = ccm[4]
b1 = ccm[6]
b2 = ccm[7]
'''
COLOR MATRIX LOOKS AS BELOW
R1 R2 R3 Rval Outr
G1 G2 G3 * Gval = G
B1 B2 B3 Bval B
Will be optimising 6 elements and working out the third element using 1-r1-r2 = r3
'''
x0 = [r1, r2, g1, g2, b1, b2]
'''
We use our old CCM as the initial guess for the program to find the
optimised matrix
'''
result = minimize(guess, x0, args=(r, g, b, m_lab), tol=0.01)
'''
This produces a color matrix which has the lowest delta E possible,
based off the input data. Note it is impossible for this to reach
zero since the input data is imperfect
'''
Cam.log += ("\n \n Optimised Matrix Below: \n \n")
[r1, r2, g1, g2, b1, b2] = result.x
# The new, optimised color correction matrix values
optimised_ccm = [r1, r2, (1 - r1 - r2), g1, g2, (1 - g1 - g2), b1, b2, (1 - b1 - b2)]
# This is the optimised Color Matrix (preserving greys by summing rows up to 1)
Cam.log += str(optimised_ccm)
Cam.log += "\n Old Color Correction Matrix Below \n"
Cam.log += str(ccm)
formatted_ccm = np.array(original_ccm).reshape((3, 3))
'''
below is a whole load of code that then applies the latest color
matrix, and returns LAB values for color. This can then be used
to calculate the final delta E
'''
optimised_ccm_rgb = [] # Original Color Corrected Matrix RGB / LAB
optimised_ccm_lab = []
formatted_optimised_ccm = np.array(optimised_ccm).reshape((3, 3))
after_gamma_rgb = []
after_gamma_lab = []
for RGB in zip(r, g, b):
ccm_applied_rgb = np.dot(formatted_ccm, (np.array(RGB) / 256))
optimised_ccm_rgb.append(gamma(ccm_applied_rgb))
optimised_ccm_lab.append(colors.RGB_to_LAB(ccm_applied_rgb))
optimised_ccm_applied_rgb = np.dot(formatted_optimised_ccm, np.array(RGB) / 256)
after_gamma_rgb.append(gamma(optimised_ccm_applied_rgb))
after_gamma_lab.append(colors.RGB_to_LAB(optimised_ccm_applied_rgb))
'''
Gamma After RGB / LAB - not used in calculations, only used for visualisation
We now want to spit out some data that shows
how the optimisation has improved the color matrices
'''
Cam.log += "Here are the Improvements"
# CALCULATE WORST CASE delta e
old_worst_delta_e = 0
before_average = transform_and_evaluate(formatted_ccm, r, g, b, m_lab)
new_worst_delta_e = 0
after_average = transform_and_evaluate(formatted_optimised_ccm, r, g, b, m_lab)
for i in range(24):
old_delta_e = deltae(optimised_ccm_lab[i], m_lab[i]) # Current Old Delta E
new_delta_e = deltae(after_gamma_lab[i], m_lab[i]) # Current New Delta E
if old_delta_e > old_worst_delta_e:
old_worst_delta_e = old_delta_e
if new_delta_e > new_worst_delta_e:
new_worst_delta_e = new_delta_e
Cam.log += "Before color correction matrix was optimised, we got an average delta E of " + str(before_average) + " and a maximum delta E of " + str(old_worst_delta_e)
Cam.log += "After color correction matrix was optimised, we got an average delta E of " + str(after_average) + " and a maximum delta E of " + str(new_worst_delta_e)
visualise_macbeth_chart(m_rgb, optimised_ccm_rgb, after_gamma_rgb, str(Img.col) + str(matrix_selection_types[typenum]))
'''
The program will also save some visualisations of improvements.
Very pretty to look at. Top rectangle is ideal, Left square is
before optimisation, right square is after.
'''
"""
if a ccm has already been calculated for that temperature then don't
overwrite but save both. They will then be averaged later on
""" # Now going to use optimised color matrix, optimised_ccm
if Img.col in ccm_tab.keys():
ccm_tab[Img.col].append(optimised_ccm)
else:
ccm_tab[Img.col] = [optimised_ccm]
Cam.log += '\n'
Cam.log += '\nFinished processing images'
"""
average any ccms t# SPDX-License-Identifier: GPL-2.0-or-later
#
# Copyright (C) 2022, Paul Elder <paul.elder@ideasonboard.com>
#
# libtuning.py - An infrastructure for camera tuning tools
import argparse
import libtuning as lt
import libtuning.utils as utils
from libtuning.utils import eprint
from enum import Enum, IntEnum
class Color(IntEnum):
R = 0
GR = 1
GB = 2
B = 3
class Debug(Enum):
Plot = 1
# @brief What to do with the leftover pixels after dividing them into ALSC
# sectors, when the division gradient is uniform
# @var Float Force floating point division so all sectors divide equally
# @var DistributeFront Divide the remainder equally (until running out,
# obviously) into the existing sectors, starting from the front
# @var DistributeBack Same as DistributeFront but starting from the back
class Remainder(Enum):
Float = 0
DistributeFront = 1
DistributeBack = 2
# @brief A helper class to contain a default value for a module configuration
# parameter
class Param(object):
# @var Required The value contained in this instance is irrelevant, and the
# value must be provided by the tuning configuration file.
# @var Optional If the value is not provided by the tuning configuration
# file, then the value contained in this instance will be used instead.
# @var Hardcode The value contained in this instance will be used
class Mode(Enum):
Required = 0
Optional = 1
Hardcode = 2
# @param name Name of the parameter. Shall match the name used in the
# configuration file for the parameter
# @param required Whether or not a value is required in the config
# parameter of get_value()
# @param val Default value (only relevant if mode is Optional)
def __init__(self, name: str, required: Mode, val=None):
self.name = name
self.__required = required
self.val = val
def get_value(self, config: dict):
if self.__required is self.Mode.Hardcode:
return self.val
if self.__required is self.Mode.Required and self.name not in config:
raise ValueError(f'Parameter {self.name} is required but not provided in the configuration')
return config[self.name] if self.required else self.val
@property
def required(self):
return self.__required is self.Mode.Required
# @brief Used by libtuning to auto-generate help information for the tuning
# script on the available parameters for the configuration file
# \todo Implement this
@property
def info(self):
raise NotImplementedError
class Tuner(object):
# External functions
def __init__(self, platform_name):
self.name = platform_name
self.modules = []
self.parser = None
self.generator = None
self.output_order = []
self.config = {}
self.output = {}
def add(self, module):
self.modules.append(module)
def set_input_parser(self, parser):
self.parser = parser
def set_output_formatter(self, output):
self.generator = output
def set_output_order(self, modules):
self.output_order = modules
# @brief Convert classes in self.output_order to the instances in self.modules
def _prepare_output_order(self):
output_order = self.output_order
self.output_order = []
for module_type in output_order:
modules = [module for module in self.modules if module.type == module_type.type]
if len(modules) > 1:
eprint(f'Multiple modules found for module type "{module_type.type}"')
return False
if len(modules) < 1:
eprint(f'No module found for module type "{module_type.type}"')
return False
self.output_order.append(modules[0])
return True
# \todo Validate parser and generator at Tuner construction time?
def _validate_settings(self):
if self.parser is None:
eprint('Missing parser')
return False
if self.generator is None:
eprint('Missing generator')
return False
if len(self.modules) == 0:
eprint('No modules added')
return False
if len(self.output_order) != len(self.modules):
eprint('Number of outputs does not match number of modules')
return False
return True
def _process_args(self, argv, platform_name):
parser = argparse.ArgumentParser(description=f'Camera Tuning for {platform_name}')
parser.add_argument('-i', '--input', type=str, required=True,
help='''Directory containing calibration images (required).
Images for ALSC must be named "alsc_{Color Temperature}k_1[u].dng",
and all other images must be named "{Color Temperature}k_{Lux Level}l.dng"''')
parser.add_argument('-o', '--output', type=str, required=True,
help='Output file (required)')
# It is not our duty to scan all modules to figure out their default
# options, so simply return an empty configuration if none is provided.
parser.add_argument('-c', '--config', type=str, default='',
help='Config file (optional)')
# \todo Check if we really need this or if stderr is good enough, or if
# we want a better logging infrastructure with log levels
parser.add_argument('-l', '--log', type=str, default=None,
help='Output log file (optional)')
return parser.parse_args(argv[1:])
def run(self, argv):
args = self._process_args(argv, self.name)
if args is None:
return -1
if not self._validate_settings():
return -1
if not self._prepare_output_order():
return -1
if len(args.config) > 0:
self.config, disable = self.parser.parse(args.config, self.modules)
else:
self.config = {'general': {}}
disable = []
# Remove disabled modules
for module in disable:
if module in self.modules:
self.modules.remove(module)
for module in self.modules:
if not module.validate_config(self.config):
eprint(f'Config is invalid for module {module.type}')
return -1
has_lsc = any(isinstance(m, lt.modules.lsc.LSC) for m in self.modules)
# Only one LSC module allowed
has_only_lsc = has_lsc and len(self.modules) == 1
images = utils.load_images(args.input, self.config, not has_only_lsc, has_lsc)
if images is None or len(images) == 0:
eprint(f'No images were found, or able to load')
return -1
# Do the tuning
for module in self.modules:
out = module.process(self.config, images, self.output)
if out is None:
eprint(f'Module {module.name} failed to process, aborting')
break
self.output[module] = out
self.generator.write(args.output, self.output, self.output_order)
return 0