swscale: use 16-bit intermediate precision for RGB/XYZ conversion

The current logic uses 12-bit linear light math, which is woefully insufficient
and leads to nasty postarization artifacts. This patch simply switches the
internal logic to 16-bit precision.

This raises the memory requirement of these tables from 32 kB to 272 kB.

All relevant FATE tests updated for improved accuracy.

Fixes: #4829
Signed-off-by: Niklas Haas <git@haasn.dev>
Sponsored-by: Sovereign Tech Fund

libswscale/swscale.c

@@ -773,10 +773,10 @@ void ff_xyz12Torgb48(const SwsInternal *c, uint8_t *dst, int dst_stride,
                 c->xyz2rgb_matrix[2][1] * y +
                 c->xyz2rgb_matrix[2][2] * z >> 12;
 
-            // limit values to 12-bit depth
+            // limit values to 16-bit depth
-            r = av_clip_uintp2(r, 12);
+            r = av_clip_uint16(r);
-            g = av_clip_uintp2(g, 12);
+            g = av_clip_uint16(g);
-            b = av_clip_uintp2(b, 12);
+            b = av_clip_uint16(b);
 
             // convert from sRGBlinear to RGB and scale from 12bit to 16bit
             if (desc->flags & AV_PIX_FMT_FLAG_BE) {
@@ -832,10 +832,10 @@ void ff_rgb48Toxyz12(const SwsInternal *c, uint8_t *dst, int dst_stride,
                 c->rgb2xyz_matrix[2][1] * g +
                 c->rgb2xyz_matrix[2][2] * b >> 12;
 
-            // limit values to 12-bit depth
+            // limit values to 16-bit depth
-            x = av_clip_uintp2(x, 12);
+            x = av_clip_uint16(x);
-            y = av_clip_uintp2(y, 12);
+            y = av_clip_uint16(y);
-            z = av_clip_uintp2(z, 12);
+            z = av_clip_uint16(z);
 
             // convert from XYZlinear to X'Y'Z' and scale from 12bit to 16bit
             if (desc->flags & AV_PIX_FMT_FLAG_BE) {

libswscale/swscale_internal.h

@@ -547,10 +547,10 @@ struct SwsInternal {
 /* pre defined color-spaces gamma */
 #define XYZ_GAMMA (2.6f)
 #define RGB_GAMMA (2.2f)
-    int16_t *xyzgamma;
+    uint16_t *xyzgamma;
-    int16_t *rgbgamma;
+    uint16_t *rgbgamma;
-    int16_t *xyzgammainv;
+    uint16_t *xyzgammainv;
-    int16_t *rgbgammainv;
+    uint16_t *rgbgammainv;
     int16_t xyz2rgb_matrix[3][4];
     int16_t rgb2xyz_matrix[3][4];
 

libswscale/utils.c

@@ -951,7 +951,8 @@ static void fill_xyztables(SwsInternal *c)
         {1689, 1464,  739},
         { 871, 2929,  296},
         {  79,  488, 3891} };
-    static int16_t xyzgamma_tab[4096], rgbgamma_tab[4096], xyzgammainv_tab[4096], rgbgammainv_tab[4096];
+    static uint16_t xyzgamma_tab[4096],  rgbgammainv_tab[4096];
+    static uint16_t rgbgamma_tab[65536], xyzgammainv_tab[65536];
 
     memcpy(c->xyz2rgb_matrix, xyz2rgb_matrix, sizeof(c->xyz2rgb_matrix));
     memcpy(c->rgb2xyz_matrix, rgb2xyz_matrix, sizeof(c->rgb2xyz_matrix));
@@ -960,15 +961,19 @@ static void fill_xyztables(SwsInternal *c)
     c->xyzgammainv = xyzgammainv_tab;
     c->rgbgammainv = rgbgammainv_tab;
 
-    if (rgbgamma_tab[4095])
+    if (xyzgamma_tab[4095])
         return;
 
-    /* set gamma vectors */
+    /* set input gamma vectors */
     for (i = 0; i < 4096; i++) {
-        xyzgamma_tab[i] = lrint(pow(i / 4095.0, xyzgamma) * 4095.0);
+        xyzgamma_tab[i] = lrint(pow(i / 4095.0, xyzgamma) * 65535.0);
-        rgbgamma_tab[i] = lrint(pow(i / 4095.0, rgbgamma) * 4095.0);
+        rgbgammainv_tab[i] = lrint(pow(i / 4095.0, rgbgammainv) * 65535.0);
-        xyzgammainv_tab[i] = lrint(pow(i / 4095.0, xyzgammainv) * 4095.0);
+    }
-        rgbgammainv_tab[i] = lrint(pow(i / 4095.0, rgbgammainv) * 4095.0);
+
+    /* set output gamma vectors */
+    for (i = 0; i < 65536; i++) {
+        rgbgamma_tab[i] = lrint(pow(i / 65535.0, rgbgamma) * 4095.0);
+        xyzgammainv_tab[i] = lrint(pow(i / 65535.0, xyzgammainv) * 4095.0);
     }
 }
 

tests/ref/fate/filter-pixdesc-xyz12be

@@ -1 +1 @@
-pixdesc-xyz12be     4ec824668b9753e26c1bccffca866e27
+pixdesc-xyz12be     1508a33dea936c45d9ee13f7743af00d

tests/ref/fate/filter-pixdesc-xyz12le

@@ -1 +1 @@
-pixdesc-xyz12le     88d2563589044a3e28f6cde9a43599f9
+pixdesc-xyz12le     da2d1326fa5747a7f6ce5ac1e1494aea

tests/ref/fate/filter-pixfmts-copy

@@ -111,8 +111,8 @@ xv36be              9f556ee59a672fd8725f0bb36ce3e4b0
 xv36le              e08dcbde02f1c28a3554f372ad1278e2
 xv48be              ce34993b4b4411bba1d852b9b86aa39e
 xv48le              df913a7e61b162aa98303e5393e60c63
-xyz12be             a1ef56bf746d71f59669c28e48fc8450
+xyz12be             f257f86373207af8aed0a1a05171df3b
-xyz12le             831ff03c1ba4ef19374686f16a064d8c
+xyz12le             7922f99edc44a2c26a25becbea9914cc
 y210le              04e9487b6cce38e7531437e946cdd586
 y212le              825768be8fe92708ae80be84855066ed
 y216le              0e99aeddfee304e72d525d72998d9e9b

tests/ref/fate/filter-pixfmts-crop

@@ -108,8 +108,8 @@ xv36be              23b6f253fcb375e4145cfcb562268c5f
 xv36le              778286003497f92b84d0bd8258d6b85d
 xv48be              c90889b2cf54cc78bd58e8c47d4eb791
 xv48le              2c15c1254449ec5f9135ae61bdf4e1d5
-xyz12be             cb4571f9aaa7b59f999ef327276104b7
+xyz12be             e2f9f6a1ec205ab675a5a1c9521dfa6c
-xyz12le             cd6aae8d26b18bdb4b9d068586276d91
+xyz12le             fea1da11c07736303b139bc52b7d4759
 ya16be              071add03126a11dc6a06209e9b409f8d
 ya16le              b723211dc0647c944768c6e45e066b36
 ya8                 51a8dd297e35d40b06d3ebe8f4717895

tests/ref/fate/filter-pixfmts-field

@@ -111,8 +111,8 @@ xv36be              bcc7bda2d0a5d43db4464af6a4cb5d65
 xv36le              ba99f258370f2a56993e8760e6b30194
 xv48be              2abcd986a34789ba4310be3969020d0d
 xv48le              f6f2e33f260f48334197538f3331f7bc
-xyz12be             d2fa69ec91d3ed862f2dac3f8e7a3437
+xyz12be             3b6eb75517263b9e54b9bfa869de394f
-xyz12le             02bccd5e0b6824779a1f848b0ea3e3b5
+xyz12le             27d1d6a488cbc5d53e8d12fa0e162ddb
 y210le              4c2fba1dc40322584977d15dd07c9146
 y212le              ac2a47c45187dd54d0f55293cbffd954
 y216le              e65b5bfae1b40edbbed2012e9cd45e31

tests/ref/fate/filter-pixfmts-fieldorder

@@ -100,8 +100,8 @@ xv36be              962386c88268f4382004c3a7a82c5eb8
 xv36le              bcceffc985aaa8414c4b8072aa0889bd
 xv48be              4d6e4004b03767f12df8bb4e76c98ddf
 xv48le              9e94d82461a2131063157ac0dbe9467b
-xyz12be             15f5cda71de5fef9cec5e75e3833b6bc
+xyz12be             ba6928f85c202cd77e216934f6bf0698
-xyz12le             7be6c8781f38c21a6b8f602f62ca31e6
+xyz12le             964680cd3f3db8a7ef5510f90196961a
 y210le              22b1a02a39c4b325726bf8793bf1e8f2
 y212le              2f08fb195b948056c844acb1eee8d649
 y216le              360cb98ac80b13d3a8ec61c9f1ff3bac

tests/ref/fate/filter-pixfmts-hflip

@@ -108,8 +108,8 @@ xv36be              98f578df965eed369f46cb135e2d1345
 xv36le              e478b4b54698beb3ce1b9a2dd691d544
 xv48be              e030a2c7b1b600cfacb691b6e90c2e3d
 xv48le              fbd7f8c65cd6fc9f9108dc9a1f977dc3
-xyz12be             25f90259ff8a226befdaec3dfe82996e
+xyz12be             3c50a51a3c486a0c6853e4bbbcf3f244
-xyz12le             926c0791d59aaff61b2778e8ada3316d
+xyz12le             e020897d826ea20ded16f30ea1eb018d
 ya16be              70fa41c32ecaf3370edc38add6096db2
 ya16le              3b2c20f9e80717628ced6c6468507f63
 ya8                 4ad5920716de3d2fbbc49f95adb60345

tests/ref/fate/filter-pixfmts-il

@@ -110,8 +110,8 @@ xv36be              3bbb949278ea55cc947ee03bd9c27c2d
 xv36le              102c0e817d375ddd6b2cfbb4262dec95
 xv48be              4d7376651fb7b3e84d00abad6c785aad
 xv48le              a1a8ff16d9a864568e5e557734bf3d6d
-xyz12be             7c7d54c55f136cbbc50b18029f3be0b3
+xyz12be             b7d50e283360bf69fd661369110b26ef
-xyz12le             090ba6b1170baf2b1358b43b971d33b0
+xyz12le             d5b1d45c3a136bb3d04f70a619c86c8d
 y210le              d4cf9b53cd7ff22f087743d483e88480
 y212le              d5a2b4677ddb4a3bc3e5cd5cbb20f426
 y216le              9e44c6d76b09bcbe71738423b4b3d67a

tests/ref/fate/filter-pixfmts-null

@@ -111,8 +111,8 @@ xv36be              9f556ee59a672fd8725f0bb36ce3e4b0
 xv36le              e08dcbde02f1c28a3554f372ad1278e2
 xv48be              ce34993b4b4411bba1d852b9b86aa39e
 xv48le              df913a7e61b162aa98303e5393e60c63
-xyz12be             a1ef56bf746d71f59669c28e48fc8450
+xyz12be             f257f86373207af8aed0a1a05171df3b
-xyz12le             831ff03c1ba4ef19374686f16a064d8c
+xyz12le             7922f99edc44a2c26a25becbea9914cc
 y210le              04e9487b6cce38e7531437e946cdd586
 y212le              825768be8fe92708ae80be84855066ed
 y216le              0e99aeddfee304e72d525d72998d9e9b

tests/ref/fate/filter-pixfmts-scale

@@ -111,8 +111,8 @@ xv36be              4d084adca0228d7750d1e2e877e0d79b
 xv36le              de9c74e94dc19c828e1572aa283d8aca
 xv48be              9e58d1a045df100b0dec116e13be5b4e
 xv48le              fd873d53609b2fbdfe99470f515a234c
-xyz12be             c7ba8345998c0141ddc079cdd29b1a40
+xyz12be             f1905012d9b845306d9bef68d0fc81d5
-xyz12le             95f5d3a0de834cc495c9032a14987cde
+xyz12le             cfe1a3bbe391d83d381f590a00e1a16d
 y210le              7c2aef142d88ab343ec01acd45f38466
 y212le              39a3c0c843041ad4501b3107dd91ef17
 y216le              17be2999e97d36b8ed903f07ef428c09

tests/ref/fate/filter-pixfmts-transpose

@@ -100,8 +100,8 @@ xv36be              2261a0e3db5ee607d37f68d19704ae15
 xv36le              9202133de91bf64c76ca27d5cd0c816a
 xv48be              14373b7fe123225689e76fe2ce43fb93
 xv48le              319df9724a067c7b5efa215f9f54d127
-xyz12be             68e5cba640f6e4ef72dff950e88b5342
+xyz12be             69737aceb508a73365664d04c340dd3b
-xyz12le             8b6b6a6db4d7561e80db88ccaecce7a9
+xyz12le             70dd5fab9d8383b0d2e772b3b6569df4
 ya16be              6098f7d2ede0aab6b2d93d2b4f4d915a
 ya16le              1fae63e3e320ba9e6c12c29a48c44eff
 ya8                 d4b7a62f80681fa44c977ff3a64f4ce4

tests/ref/fate/filter-pixfmts-vflip

@@ -111,8 +111,8 @@ xv36be              c0272372d3e1a59adb3931ee433a5d5b
 xv36le              ffe6ab75ebc09134c3451f8f6ef0d501
 xv48be              bdfc3217ae456b370dbdcf4d52606a3f
 xv48le              53dbebab73a66539c04644ef56dc6bbb
-xyz12be             23fa9fb36d49dce61e284d41b83e0e6b
+xyz12be             1bffa153a4a3ae61fd18e370f95161d9
-xyz12le             ef73e6d1f932a9a355df1eedd628394f
+xyz12le             a0e93443826621a9d6c48354d949898a
 y210le              f8847bedd3ae6e1c0cf84a823f275e31
 y212le              c801725ae31e3b8f5be269359d49f191
 y216le              985db498aedf3fb1c547ad07442b7258

tests/ref/pixfmt/gbrp-xyz12le

@@ -1,2 +1,2 @@
-86b542ae625ef5a61cef6b0b703f553f *tests/data/pixfmt/gbrp-xyz12le.yuv
+543fe8482e02d14790b2dc48ec0812e8 *tests/data/pixfmt/gbrp-xyz12le.yuv
 7603200 tests/data/pixfmt/gbrp-xyz12le.yuv

tests/ref/pixfmt/gbrp10-xyz12le

@@ -1,2 +1,2 @@
-64eb126dca4cf2bae0b5c7677ed7f266 *tests/data/pixfmt/gbrp10-xyz12le.yuv
+68d9a907c0b823c6af7b8cc253b2d148 *tests/data/pixfmt/gbrp10-xyz12le.yuv
 15206400 tests/data/pixfmt/gbrp10-xyz12le.yuv

tests/ref/pixfmt/gbrp12-xyz12le

@@ -1,2 +1,2 @@
-6780af8d13a4d8dcbf479d86f1099e23 *tests/data/pixfmt/gbrp12-xyz12le.yuv
+6aeb56c9f41206c8f4a2439b9b755e81 *tests/data/pixfmt/gbrp12-xyz12le.yuv
 15206400 tests/data/pixfmt/gbrp12-xyz12le.yuv

tests/ref/pixfmt/rgb24-xyz12le

@@ -1,2 +1,2 @@
-460b2b2fc5611d1125c87d5f7716511c *tests/data/pixfmt/rgb24-xyz12le.yuv
+be4b641ec6f5b3c534d6b69e3144c8ca *tests/data/pixfmt/rgb24-xyz12le.yuv
 7603200 tests/data/pixfmt/rgb24-xyz12le.yuv

tests/ref/pixfmt/rgb48-xyz12le

@@ -1,2 +1,2 @@
-16a15582e2fe7ff56951ca9d06d3a7a3 *tests/data/pixfmt/rgb48-xyz12le.yuv
+380feb3972a3fd9f4a839585adda3f9b *tests/data/pixfmt/rgb48-xyz12le.yuv
 15206400 tests/data/pixfmt/rgb48-xyz12le.yuv

tests/ref/pixfmt/xyz12le

@@ -1,2 +1,2 @@
-a5a4f73b21d5e2b07789f41d84265d9a *tests/data/pixfmt/xyz12le.yuv
+47b890678bb9195107c2f7afb2344bbf *tests/data/pixfmt/xyz12le.yuv
 304128 tests/data/pixfmt/xyz12le.yuv

tests/ref/pixfmt/yuv444p-xyz12le

@@ -1,2 +1,2 @@
-a0acdd9c25a84c8698d0dc1e52950c8d *tests/data/pixfmt/yuv444p-xyz12le.yuv
+54a3bdec56e545c9ae2782e78b010f76 *tests/data/pixfmt/yuv444p-xyz12le.yuv
 7603200 tests/data/pixfmt/yuv444p-xyz12le.yuv

tests/ref/pixfmt/yuv444p10-xyz12le

@@ -1,2 +1,2 @@
-52b4ba6b111bd781c99ed8350e3b25e1 *tests/data/pixfmt/yuv444p10-xyz12le.yuv
+749b383b2fb0920de0a35010fca42acb *tests/data/pixfmt/yuv444p10-xyz12le.yuv
 15206400 tests/data/pixfmt/yuv444p10-xyz12le.yuv

tests/ref/pixfmt/yuv444p12-xyz12le

@@ -1,2 +1,2 @@
-7dd2de39e7bc112ebd94f39284b16f06 *tests/data/pixfmt/yuv444p12-xyz12le.yuv
+3e19343dbcf62d1aa817c30643408439 *tests/data/pixfmt/yuv444p12-xyz12le.yuv
 15206400 tests/data/pixfmt/yuv444p12-xyz12le.yuv