v4l2: v4l2_compat: Intercept open64, openat64, and mmap64

Some applications (eg. Firefox, Google Chrome, Skype) use open64,
openat64, and mmap64 instead of their non-64 versions that we currently
intercept. Intercept these calls as well. _LARGEFILE64_SOURCE needs to
be set so that the 64-bit symbols are available and not synonymous to
the non-64-bit versions on 64-bit systems.

Also, since we set _FILE_OFFSET_BITS to 32 to force the various open and
mmap symbols that we export to not be the 64-bit versions, our dlsym to
get the original open and mmap calls will not automatically be converted
to their 64-bit versions. Since we intercept both 32-bit and 64-bit
versions of open and mmap, we should be using the 64-bit version to
service both. Fetch the 64-bit versions of openat and mmap directly.

musl defines the 64-bit symbols as macros that are equivalent to the
non-64-bit symbols, so we put compile guards that check if the 64-bit
symbols are defined.

Signed-off-by: Paul Elder <paul.elder@ideasonboard.com>
Reviewed-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>
Tested-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com> # Compile with musl
Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>

6 files changed, 53 insertions, 7 deletions

diff --git a/src/v4l2/meson.build b/src/v4l2/meson.build
index 0fb941ea..f2e4aaf2 100644
--- a/src/v4l2/meson.build
+++ b/src/v4l2/meson.build
@@ -15,6 +15,7 @@ v4l2_compat_cpp_args = [
     # file operations, disable transparent large file support.
     '-U_FILE_OFFSET_BITS',
     '-D_FILE_OFFSET_BITS=32',
+    '-D_LARGEFILE64_SOURCE',
     '-fvisibility=hidden',
 ]
 
diff --git a/src/v4l2/v4l2_camera_proxy.cpp b/src/v4l2/v4l2_camera_proxy.cpp
index 19b300f4..bf47aa7e 100644
--- a/src/v4l2/v4l2_camera_proxy.cpp
+++ b/src/v4l2/v4l2_camera_proxy.cpp
@@ -76,7 +76,7 @@ void V4L2CameraProxy::close()
 }
 
 void *V4L2CameraProxy::mmap(void *addr, size_t length, int prot, int flags,
-			    off_t offset)
+			    off64_t offset)
 {
 	LOG(V4L2Compat, Debug) << "Servicing mmap";
 
diff --git a/src/v4l2/v4l2_camera_proxy.h b/src/v4l2/v4l2_camera_proxy.h
index 7c65c886..27d3e507 100644
--- a/src/v4l2/v4l2_camera_proxy.h
+++ b/src/v4l2/v4l2_camera_proxy.h
@@ -11,6 +11,7 @@
 #include <linux/videodev2.h>
 #include <map>
 #include <memory>
+#include <sys/mman.h>
 #include <sys/types.h>
 #include <vector>
 
@@ -28,7 +29,7 @@ public:
 	int open(bool nonBlocking);
 	void dup();
 	void close();
-	void *mmap(void *addr, size_t length, int prot, int flags, off_t offset);
+	void *mmap(void *addr, size_t length, int prot, int flags, off64_t offset);
 	int munmap(void *addr, size_t length);
 
 	int ioctl(unsigned long request, void *arg);
diff --git a/src/v4l2/v4l2_compat.cpp b/src/v4l2/v4l2_compat.cpp
index 2a9a1762..fd73381d 100644
--- a/src/v4l2/v4l2_compat.cpp
+++ b/src/v4l2/v4l2_compat.cpp
@@ -44,6 +44,23 @@ LIBCAMERA_PUBLIC int __open_2(const char *path, int oflag)
 	return open(path, oflag);
 }
 
+#ifndef open64
+LIBCAMERA_PUBLIC int open64(const char *path, int oflag, ...)
+{
+	mode_t mode = 0;
+	if (oflag & O_CREAT || oflag & O_TMPFILE)
+		extract_va_arg(mode_t, mode, oflag);
+
+	return V4L2CompatManager::instance()->openat(AT_FDCWD, path,
+						     oflag | O_LARGEFILE, mode);
+}
+
+LIBCAMERA_PUBLIC int __open64_2(const char *path, int oflag)
+{
+	return open(path, oflag);
+}
+#endif
+
 LIBCAMERA_PUBLIC int openat(int dirfd, const char *path, int oflag, ...)
 {
 	mode_t mode = 0;
@@ -58,6 +75,23 @@ LIBCAMERA_PUBLIC int __openat_2(int dirfd, const char *path, int oflag)
 	return openat(dirfd, path, oflag);
 }
 
+#ifndef openat64
+LIBCAMERA_PUBLIC int openat64(int dirfd, const char *path, int oflag, ...)
+{
+	mode_t mode = 0;
+	if (oflag & O_CREAT || oflag & O_TMPFILE)
+		extract_va_arg(mode_t, mode, oflag);
+
+	return V4L2CompatManager::instance()->openat(dirfd, path,
+						     oflag | O_LARGEFILE, mode);
+}
+
+LIBCAMERA_PUBLIC int __openat64_2(int dirfd, const char *path, int oflag)
+{
+	return openat(dirfd, path, oflag);
+}
+#endif
+
 LIBCAMERA_PUBLIC int dup(int oldfd)
 {
 	return V4L2CompatManager::instance()->dup(oldfd);
@@ -75,6 +109,15 @@ LIBCAMERA_PUBLIC void *mmap(void *addr, size_t length, int prot, int flags,
 						   fd, offset);
 }
 
+#ifndef mmap64
+LIBCAMERA_PUBLIC void *mmap64(void *addr, size_t length, int prot, int flags,
+			      int fd, off64_t offset)
+{
+	return V4L2CompatManager::instance()->mmap(addr, length, prot, flags,
+						   fd, offset);
+}
+#endif
+
 LIBCAMERA_PUBLIC int munmap(void *addr, size_t length)
 {
 	return V4L2CompatManager::instance()->munmap(addr, length);
diff --git a/src/v4l2/v4l2_compat_manager.cpp b/src/v4l2/v4l2_compat_manager.cpp
index 56533c4f..8da33164 100644
--- a/src/v4l2/v4l2_compat_manager.cpp
+++ b/src/v4l2/v4l2_compat_manager.cpp
@@ -39,11 +39,11 @@ void get_symbol(T &func, const char *name)
 V4L2CompatManager::V4L2CompatManager()
 	: cm_(nullptr)
 {
-	get_symbol(fops_.openat, "openat");
+	get_symbol(fops_.openat, "openat64");
 	get_symbol(fops_.dup, "dup");
 	get_symbol(fops_.close, "close");
 	get_symbol(fops_.ioctl, "ioctl");
-	get_symbol(fops_.mmap, "mmap");
+	get_symbol(fops_.mmap, "mmap64");
 	get_symbol(fops_.munmap, "munmap");
 }
 
@@ -200,7 +200,7 @@ int V4L2CompatManager::close(int fd)
 }
 
 void *V4L2CompatManager::mmap(void *addr, size_t length, int prot, int flags,
-			      int fd, off_t offset)
+			      int fd, off64_t offset)
 {
 	V4L2CameraProxy *proxy = getProxy(fd);
 	if (!proxy)
diff --git a/src/v4l2/v4l2_compat_manager.h b/src/v4l2/v4l2_compat_manager.h
index 14338a59..3d4e512e 100644
--- a/src/v4l2/v4l2_compat_manager.h
+++ b/src/v4l2/v4l2_compat_manager.h
@@ -11,6 +11,7 @@
 #include <fcntl.h>
 #include <map>
 #include <memory>
+#include <sys/mman.h>
 #include <sys/types.h>
 #include <vector>
 
@@ -30,7 +31,7 @@ public:
 		using close_func_t = int (*)(int fd);
 		using ioctl_func_t = int (*)(int fd, unsigned long request, ...);
 		using mmap_func_t = void *(*)(void *addr, size_t length, int prot,
-					      int flags, int fd, off_t offset);
+					      int flags, int fd, off64_t offset);
 		using munmap_func_t = int (*)(void *addr, size_t length);
 
 		openat_func_t openat;
@@ -51,7 +52,7 @@ public:
 	int dup(int oldfd);
 	int close(int fd);
 	void *mmap(void *addr, size_t length, int prot, int flags,
-		   int fd, off_t offset);
+		   int fd, off64_t offset);
 	int munmap(void *addr, size_t length);
 	int ioctl(int fd, unsigned long request, void *arg);