From b053384ffab467ed016019bbcf8121a6f9e48fed Mon Sep 17 00:00:00 2001
From: Umang Jain <email@uajain.com>
Date: Wed, 28 Oct 2020 02:54:47 +0530
Subject: android: jpeg: post_processor_jpeg: Embed thumbnail into Exif
 metadata

Embed a Jpeg-encoded thumbnail into Exif metadata using the Thumbnailer
class that got introduced.

Introduce a helper function in Exif class for setting the thumbnail
data.

Signed-off-by: Umang Jain <email@uajain.com>
Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
[Kieran: Add todo comment, and Compression enum]
Signed-off-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
---
 src/android/jpeg/exif.cpp                | 24 ++++++++++++++++++-
 src/android/jpeg/exif.h                  |  7 ++++++
 src/android/jpeg/post_processor_jpeg.cpp | 40 ++++++++++++++++++++++++++++++++
 src/android/jpeg/post_processor_jpeg.h   |  8 ++++++-
 4 files changed, 77 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/android/jpeg/exif.cpp b/src/android/jpeg/exif.cpp
index d21534a4..33b3fa7f 100644
--- a/src/android/jpeg/exif.cpp
+++ b/src/android/jpeg/exif.cpp
@@ -75,8 +75,16 @@ Exif::~Exif()
 	if (exifData_)
 		free(exifData_);
 
-	if (data_)
+	if (data_) {
+		/*
+		 * Reset thumbnail data to avoid getting double-freed by
+		 * libexif. It is owned by the caller (i.e. PostProcessorJpeg).
+		 */
+		data_->data = nullptr;
+		data_->size = 0;
+
 		exif_data_unref(data_);
+	}
 
 	if (mem_)
 		exif_mem_unref(mem_);
@@ -268,6 +276,20 @@ void Exif::setOrientation(int orientation)
 	setShort(EXIF_IFD_0, EXIF_TAG_ORIENTATION, value);
 }
 
+/*
+ * The thumbnail data should remain valid until the Exif object is destroyed.
+ * Failing to do so, might result in no thumbnail data being set even after a
+ * call to Exif::setThumbnail().
+ */
+void Exif::setThumbnail(Span<const unsigned char> thumbnail,
+			Compression compression)
+{
+	data_->data = const_cast<unsigned char *>(thumbnail.data());
+	data_->size = thumbnail.size();
+
+	setShort(EXIF_IFD_0, EXIF_TAG_COMPRESSION, compression);
+}
+
 [[nodiscard]] int Exif::generate()
 {
 	if (exifData_) {
diff --git a/src/android/jpeg/exif.h b/src/android/jpeg/exif.h
index 12c27b6f..5cab4559 100644
--- a/src/android/jpeg/exif.h
+++ b/src/android/jpeg/exif.h
@@ -21,11 +21,18 @@ public:
 	Exif();
 	~Exif();
 
+	enum Compression {
+		None = 1,
+		JPEG = 6,
+	};
+
 	void setMake(const std::string &make);
 	void setModel(const std::string &model);
 
 	void setOrientation(int orientation);
 	void setSize(const libcamera::Size &size);
+	void setThumbnail(libcamera::Span<const unsigned char> thumbnail,
+			  Compression compression);
 	void setTimestamp(time_t timestamp);
 
 	libcamera::Span<const uint8_t> data() const { return { exifData_, size_ }; }
diff --git a/src/android/jpeg/post_processor_jpeg.cpp b/src/android/jpeg/post_processor_jpeg.cpp
index 93acfe52..b9b9dc00 100644
--- a/src/android/jpeg/post_processor_jpeg.cpp
+++ b/src/android/jpeg/post_processor_jpeg.cpp
@@ -39,11 +39,45 @@ int PostProcessorJpeg::configure(const StreamConfiguration &inCfg,
 	}
 
 	streamSize_ = outCfg.size;
+
+	thumbnailer_.configure(inCfg.size, inCfg.pixelFormat);
+	StreamConfiguration thCfg = inCfg;
+	thCfg.size = thumbnailer_.size();
+	if (thumbnailEncoder_.configure(thCfg) != 0) {
+		LOG(JPEG, Error) << "Failed to configure thumbnail encoder";
+		return -EINVAL;
+	}
+
 	encoder_ = std::make_unique<EncoderLibJpeg>();
 
 	return encoder_->configure(inCfg);
 }
 
+void PostProcessorJpeg::generateThumbnail(const FrameBuffer &source,
+					  std::vector<unsigned char> *thumbnail)
+{
+	/* Stores the raw scaled-down thumbnail bytes. */
+	std::vector<unsigned char> rawThumbnail;
+
+	thumbnailer_.createThumbnail(source, &rawThumbnail);
+
+	if (!rawThumbnail.empty()) {
+		/*
+		 * \todo Avoid value-initialization of all elements of the
+		 * vector.
+		 */
+		thumbnail->resize(rawThumbnail.size());
+
+		int jpeg_size = thumbnailEncoder_.encode(rawThumbnail,
+							 *thumbnail, {});
+		thumbnail->resize(jpeg_size);
+
+		LOG(JPEG, Debug)
+			<< "Thumbnail compress returned "
+			<< jpeg_size << " bytes";
+	}
+}
+
 int PostProcessorJpeg::process(const FrameBuffer &source,
 			       Span<uint8_t> destination,
 			       CameraMetadata *metadata)
@@ -64,6 +98,12 @@ int PostProcessorJpeg::process(const FrameBuffer &source,
 	 * second, it is good enough.
 	 */
 	exif.setTimestamp(std::time(nullptr));
+
+	std::vector<unsigned char> thumbnail;
+	generateThumbnail(source, &thumbnail);
+	if (!thumbnail.empty())
+		exif.setThumbnail(thumbnail, Exif::Compression::JPEG);
+
 	if (exif.generate() != 0)
 		LOG(JPEG, Error) << "Failed to generate valid EXIF data";
 
diff --git a/src/android/jpeg/post_processor_jpeg.h b/src/android/jpeg/post_processor_jpeg.h
index 3706cec2..5afa831c 100644
--- a/src/android/jpeg/post_processor_jpeg.h
+++ b/src/android/jpeg/post_processor_jpeg.h
@@ -8,12 +8,13 @@
 #define __ANDROID_POST_PROCESSOR_JPEG_H__
 
 #include "../post_processor.h"
+#include "encoder_libjpeg.h"
+#include "thumbnailer.h"
 
 #include <libcamera/geometry.h>
 
 #include "libcamera/internal/buffer.h"
 
-class Encoder;
 class CameraDevice;
 
 class PostProcessorJpeg : public PostProcessor
@@ -28,9 +29,14 @@ public:
 		    CameraMetadata *metadata) override;
 
 private:
+	void generateThumbnail(const libcamera::FrameBuffer &source,
+			       std::vector<unsigned char> *thumbnail);
+
 	CameraDevice *const cameraDevice_;
 	std::unique_ptr<Encoder> encoder_;
 	libcamera::Size streamSize_;
+	EncoderLibJpeg thumbnailEncoder_;
+	Thumbnailer thumbnailer_;
 };
 
 #endif /* __ANDROID_POST_PROCESSOR_JPEG_H__ */
-- 
cgit v1.2.1