sws: GBRP9, GBRP10, and GBRP16 output support

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>

libswscale/output.c

@@ -1269,8 +1269,11 @@ yuv2gbrp_full_X_c(SwsContext *c, const int16_t *lumFilter,
                   const int16_t **alpSrc, uint8_t **dest,
                   int dstW, int y)
 {
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(c->dstFormat);
     int i;
     int hasAlpha = 0;
+    uint16_t **dest16 = (uint16_t**)dest;
+    int SH = 22 + 7 - desc->comp[0].depth_minus1;
 
     for (i = 0; i < dstW; i++) {
         int j;
@@ -1316,9 +1319,22 @@ yuv2gbrp_full_X_c(SwsContext *c, const int16_t *lumFilter,
             B = av_clip_uintp2(B, 30);
         }
 
-        dest[0][i] = G >> 22;
+        if (SH != 22) {
-        dest[1][i] = B >> 22;
+            dest16[0][i] = G >> SH;
-        dest[2][i] = R >> 22;
+            dest16[1][i] = B >> SH;
+            dest16[2][i] = R >> SH;
+        } else {
+            dest[0][i] = G >> 22;
+            dest[1][i] = B >> 22;
+            dest[2][i] = R >> 22;
+        }
+    }
+    if (SH != 22 && (!isBE(c->dstFormat)) != (!HAVE_BIGENDIAN)) {
+        for (i = 0; i < dstW; i++) {
+            dest16[0][i] = av_bswap16(dest16[0][i]);
+            dest16[1][i] = av_bswap16(dest16[1][i]);
+            dest16[2][i] = av_bswap16(dest16[2][i]);
+        }
     }
 }
 
@@ -1417,6 +1433,12 @@ av_cold void ff_sws_init_output_funcs(SwsContext *c,
             *yuv2packedX = yuv2bgr24_full_X_c;
             break;
         case AV_PIX_FMT_GBRP:
+        case AV_PIX_FMT_GBRP9BE:
+        case AV_PIX_FMT_GBRP9LE:
+        case AV_PIX_FMT_GBRP10BE:
+        case AV_PIX_FMT_GBRP10LE:
+        case AV_PIX_FMT_GBRP16BE:
+        case AV_PIX_FMT_GBRP16LE:
             *yuv2anyX = yuv2gbrp_full_X_c;
             break;
         }

libswscale/utils.c

@@ -164,12 +164,12 @@ static const FormatEntry format_entries[AV_PIX_FMT_NB] = {
     [AV_PIX_FMT_YUV444P10BE] = { 1, 1 },
     [AV_PIX_FMT_YUV444P10LE] = { 1, 1 },
     [AV_PIX_FMT_GBRP]        = { 1, 1 },
-    [AV_PIX_FMT_GBRP9LE]     = { 1, 0 },
+    [AV_PIX_FMT_GBRP9LE]     = { 1, 1 },
-    [AV_PIX_FMT_GBRP9BE]     = { 1, 0 },
+    [AV_PIX_FMT_GBRP9BE]     = { 1, 1 },
-    [AV_PIX_FMT_GBRP10LE]    = { 1, 0 },
+    [AV_PIX_FMT_GBRP10LE]    = { 1, 1 },
-    [AV_PIX_FMT_GBRP10BE]    = { 1, 0 },
+    [AV_PIX_FMT_GBRP10BE]    = { 1, 1 },
-    [AV_PIX_FMT_GBRP16LE]    = { 1, 0 },
+    [AV_PIX_FMT_GBRP16LE]    = { 1, 1 },
-    [AV_PIX_FMT_GBRP16BE]    = { 1, 0 },
+    [AV_PIX_FMT_GBRP16BE]    = { 1, 1 },
 };
 
 int sws_isSupportedInput(enum AVPixelFormat pix_fmt)
@@ -938,7 +938,7 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
     getSubSampleFactors(&c->chrSrcHSubSample, &c->chrSrcVSubSample, srcFormat);
     getSubSampleFactors(&c->chrDstHSubSample, &c->chrDstVSubSample, dstFormat);
 
-    if (dstFormat == AV_PIX_FMT_GBRP) {
+    if (isPlanarRGB(dstFormat)) {
         if (!(flags & SWS_FULL_CHR_H_INT)) {
             av_log(c, AV_LOG_DEBUG,
                    "%s output is not supported with half chroma resolution, switching to full\n",
@@ -952,7 +952,7 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
      * chroma interpolation */
     if (flags & SWS_FULL_CHR_H_INT &&
         isAnyRGB(dstFormat)        &&
-        dstFormat != AV_PIX_FMT_GBRP  &&
+        !isPlanarRGB(dstFormat)    &&
         dstFormat != AV_PIX_FMT_RGBA  &&
         dstFormat != AV_PIX_FMT_ARGB  &&
         dstFormat != AV_PIX_FMT_BGRA  &&

tests/ref/lavfi/pixdesc

@@ -13,6 +13,12 @@ bgr565le            1dfdd03995c287e3c754b164bf26a355
 bgr8                24bd566170343d06fec6fccfff5abc54
 bgra                76a18a5151242fa137133f604cd624d2
 gbrp                76204621e200a3cc633012f6720c7005
+gbrp10be            2ca4a4a589a7dc461ff186913c7a69e5
+gbrp10le            46176f1fcc6e67f9862115fe373f73d3
+gbrp16be            b577122bb4fc34274b9fc2d1c6b934d0
+gbrp16le            7a8fc641745c4f363348522a687d2a08
+gbrp9be             981c2c1b0ef1791824b4c7518331bc2e
+gbrp9le             25fb915ed11d07d631e0e7b78d54bebf
 gray                db08f7f0751900347e6b8649e4164d21
 gray16be            7becf34ae825a3df3969bf4c6bfeb5e2
 gray16le            10bd87059b5c189f3caef2837f4f2b5c

tests/ref/lavfi/pixfmts_copy

@@ -13,6 +13,12 @@ bgr565le            1dfdd03995c287e3c754b164bf26a355
 bgr8                24bd566170343d06fec6fccfff5abc54
 bgra                76a18a5151242fa137133f604cd624d2
 gbrp                76204621e200a3cc633012f6720c7005
+gbrp10be            2ca4a4a589a7dc461ff186913c7a69e5
+gbrp10le            46176f1fcc6e67f9862115fe373f73d3
+gbrp16be            b577122bb4fc34274b9fc2d1c6b934d0
+gbrp16le            7a8fc641745c4f363348522a687d2a08
+gbrp9be             981c2c1b0ef1791824b4c7518331bc2e
+gbrp9le             25fb915ed11d07d631e0e7b78d54bebf
 gray                db08f7f0751900347e6b8649e4164d21
 gray16be            7becf34ae825a3df3969bf4c6bfeb5e2
 gray16le            10bd87059b5c189f3caef2837f4f2b5c

tests/ref/lavfi/pixfmts_null

@@ -13,6 +13,12 @@ bgr565le            1dfdd03995c287e3c754b164bf26a355
 bgr8                24bd566170343d06fec6fccfff5abc54
 bgra                76a18a5151242fa137133f604cd624d2
 gbrp                76204621e200a3cc633012f6720c7005
+gbrp10be            2ca4a4a589a7dc461ff186913c7a69e5
+gbrp10le            46176f1fcc6e67f9862115fe373f73d3
+gbrp16be            b577122bb4fc34274b9fc2d1c6b934d0
+gbrp16le            7a8fc641745c4f363348522a687d2a08
+gbrp9be             981c2c1b0ef1791824b4c7518331bc2e
+gbrp9le             25fb915ed11d07d631e0e7b78d54bebf
 gray                db08f7f0751900347e6b8649e4164d21
 gray16be            7becf34ae825a3df3969bf4c6bfeb5e2
 gray16le            10bd87059b5c189f3caef2837f4f2b5c

tests/ref/lavfi/pixfmts_scale

@@ -13,6 +13,12 @@ bgr565le            3a514a298c6161a071ddf9963c06509d
 bgr8                7f007fa6c153a16e808a9c51605a4016
 bgra                a5e7040f9a80cccd65e5acf2ca09ace5
 gbrp                205c50f8359cb4ba2827a7711dea2cc6
+gbrp10be            30b7f9d5ef5da474fb794743146236aa
+gbrp10le            2e9949a01fe4c38774728e34795165cc
+gbrp16be            4045de229757b7a285fea274b9599c0d
+gbrp16le            f71e77371c0c5781e46af85ea95f92a3
+gbrp9be             6bac01a7f64a381521b2149fc46f4178
+gbrp9le             4b6cba7acf7886c13db122e590ec6b1f
 gray                d7786a7d9d99ac74230cc045cab5632c
 gray16be            b554d6c1cc8da23967445be4dd3e4a86
 gray16le            715a33aa1c19cb26b14f5cc000e7a3d1

tests/ref/lavfi/pixfmts_vflip

@@ -13,6 +13,12 @@ bgr565le            fdb617533e1e7ff512ea5b6b6233e738
 bgr8                c60f93fd152c6903391d1fe9decd3547
 bgra                7f9b799fb48544e49ce93e91d7f9fca8
 gbrp                25c1bce192daefab910d51a56b52199e
+gbrp10be            6fe980f9ca94cbcdb9e01f4e906fdf19
+gbrp10le            f793a1d96d3524c6a17e53356c415c4e
+gbrp16be            6517b6c4fde50fd451e6fb16ccd204c9
+gbrp16le            5b8f07a424c3c2843b39ba5dd9803d3e
+gbrp9be             e10dc3a2566b065260a45356ef08c1cd
+gbrp9le             f25105a33f18d6bcca3205c67ec106be
 gray                30d9014a9d43b5f37e7aa64be3a3ecfc
 gray16be            6b84b85d3326182fa1217e138249edc5
 gray16le            66bb8faa09dc149734aca3c768a6d4e1