From 8936e81e3f2f160966f59e1d19103e95098db1da Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Tue, 1 Apr 2025 14:36:11 +0200 Subject: libipa: histogram: Fix quantile() calculation for fractional results The calculation of the frac variable is based solely on integers and therefore results in the fractional part being either 0 or 1. In the original code from RaspberryPi this is mitigated by casting the nominator to a double. This works for most cases, but fails when q is very small because of the quantization introduced by item being an integer. Fix both issues by doing the full calculation in double and remove the should_fail tag. Signed-off-by: Stefan Klug Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/ipa/libipa/histogram.cpp | 3 ++- test/ipa/libipa/meson.build | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ipa/libipa/histogram.cpp b/src/ipa/libipa/histogram.cpp index 10e44b54..ea042f0a 100644 --- a/src/ipa/libipa/histogram.cpp +++ b/src/ipa/libipa/histogram.cpp @@ -130,7 +130,8 @@ double Histogram::quantile(double q, uint32_t first, uint32_t last) const if (cumulative_[first + 1] == cumulative_[first]) frac = 0; else - frac = (item - cumulative_[first]) / (cumulative_[first + 1] - cumulative_[first]); + frac = (q * total() - cumulative_[first]) + / (cumulative_[first + 1] - cumulative_[first]); return first + frac; } diff --git a/test/ipa/libipa/meson.build b/test/ipa/libipa/meson.build index 83c84bd8..8c63ebd8 100644 --- a/test/ipa/libipa/meson.build +++ b/test/ipa/libipa/meson.build @@ -2,7 +2,7 @@ libipa_test = [ {'name': 'fixedpoint', 'sources': ['fixedpoint.cpp']}, - {'name': 'histogram', 'sources': ['histogram.cpp'], 'should_fail': true}, + {'name': 'histogram', 'sources': ['histogram.cpp']}, {'name': 'interpolator', 'sources': ['interpolator.cpp']}, ] -- cgit v1.2.1