From 8e0a4d2d4d1cc56a12b3c231e444fd2e94f32ce0 Mon Sep 17 00:00:00 2001 From: Andriy Gelman Date: Sun, 31 Jan 2021 14:41:47 -0500 Subject: [PATCH] avdevice/xcbgrab: don't assume xserver endianness xserver defines the endianness of the grabbed images. Use this information to set the correct pixel format. This also fixes format selection in configuration depth=32/bpp=32 with xserver on a little endian machine. Before the patch, the big endian layout 0RGB was always selected which is incorrect because BGR0 should be used. RGB24 was also incorrectly assumed (but this format was removed in xserver 1.20). The big-endian settings can be tested using docker+qemu from a little-endian machine: $ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes $ docker run --rm -it -v /tmp:/tmp powerpc64/debian /bin/bash In docker container $ apt-get update $ apt-get install xvfb $ apt-get install x11-apps To test AV_PIX_FMT_0RGB32 $ Xvfb :2 -screen 0 720x480x24 & $ export DISPLAY=:2 $ xclock -geometry 720x480 -bg green #test different colors On your host machine grab the frames using the following command. View output to check that colors are rendered correctly $ ./ffmpeg -y -f x11grab -i :2.0 -codec:v mpeg2video out.mp4 Other pixel formats can be tested by modifying how Xvfb is started in the docker container: AV_PIX_FMT_RGB565 $ Xvfb :2 -screen 0 720x480x16 AV_PIX_FMT_RGB555 $ Xvfb :2 -screen 0 720x480x15 AV_PIX_FMT_BGR24 / AV_PIX_FMT_RGB24 This is difficult to test because bpp=24 support was removed in xserver 1.20 https://lists.x.org/archives/xorg-devel/2018-February/056175.html?hmsr=joyk.com&utm_source=joyk.com&utm_medium=referral However, I was able to run previous version of Xvfb (with some modifications to force 24bpp) to check that images are rendered correctly. Reviewed-by: Carl Eugen Hoyos Signed-off-by: Andriy Gelman --- libavdevice/xcbgrab.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/libavdevice/xcbgrab.c b/libavdevice/xcbgrab.c index 95bdc8ab9d..be5d5ea2cf 100644 --- a/libavdevice/xcbgrab.c +++ b/libavdevice/xcbgrab.c @@ -513,21 +513,26 @@ static int pixfmt_from_pixmap_format(AVFormatContext *s, int depth, switch (depth) { case 32: if (fmt->bits_per_pixel == 32) - *pix_fmt = AV_PIX_FMT_0RGB; + *pix_fmt = setup->image_byte_order == XCB_IMAGE_ORDER_LSB_FIRST ? + AV_PIX_FMT_BGR0 : AV_PIX_FMT_0RGB; break; case 24: if (fmt->bits_per_pixel == 32) - *pix_fmt = AV_PIX_FMT_0RGB32; + *pix_fmt = setup->image_byte_order == XCB_IMAGE_ORDER_LSB_FIRST ? + AV_PIX_FMT_BGR0 : AV_PIX_FMT_0RGB; else if (fmt->bits_per_pixel == 24) - *pix_fmt = AV_PIX_FMT_RGB24; + *pix_fmt = setup->image_byte_order == XCB_IMAGE_ORDER_LSB_FIRST ? + AV_PIX_FMT_BGR24 : AV_PIX_FMT_RGB24; break; case 16: if (fmt->bits_per_pixel == 16) - *pix_fmt = AV_PIX_FMT_RGB565; + *pix_fmt = setup->image_byte_order == XCB_IMAGE_ORDER_LSB_FIRST ? + AV_PIX_FMT_RGB565LE : AV_PIX_FMT_RGB565BE; break; case 15: if (fmt->bits_per_pixel == 16) - *pix_fmt = AV_PIX_FMT_RGB555; + *pix_fmt = setup->image_byte_order == XCB_IMAGE_ORDER_LSB_FIRST ? + AV_PIX_FMT_RGB555LE : AV_PIX_FMT_RGB555BE; break; case 8: if (fmt->bits_per_pixel == 8)