From 7d7bacf0f1f43f5cc112977fc3b8438e3e4a4a9b Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 8 Jul 2011 14:57:07 -0700 Subject: [PATCH] swscale: fix overflow in 16-bit vertical scaling. We operated on 31-bits, but with e.g. lanczos scaling, values can add up to beyond 0x80000000, thus leading to output of zeroes. Drop one bit of precision fixes this. --- libswscale/swscale.c | 18 +++++++++--------- tests/ref/lavfi/pixdesc | 8 ++++---- tests/ref/lavfi/pixfmts_copy | 8 ++++---- tests/ref/lavfi/pixfmts_crop | 8 ++++---- tests/ref/lavfi/pixfmts_hflip | 8 ++++---- tests/ref/lavfi/pixfmts_null | 8 ++++---- tests/ref/lavfi/pixfmts_scale | 16 ++++++++-------- tests/ref/lavfi/pixfmts_vflip | 8 ++++---- 8 files changed, 41 insertions(+), 41 deletions(-) diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 527dd80e0c..db4d231e13 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -195,7 +195,7 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc, int i; uint16_t *yDest = dest[0], *uDest = dest[1], *vDest = dest[2], *aDest = CONFIG_SWSCALE_ALPHA ? dest[3] : NULL; - int shift = 15 + 16 - output_bits; + int shift = 15 + 16 - output_bits - 1; #define output_pixel(pos, val) \ if (big_endian) { \ @@ -212,24 +212,24 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc, } \ } for (i = 0; i < dstW; i++) { - int val = 1 << (30-output_bits); + int val = 1 << (30-output_bits - 1); int j; for (j = 0; j < lumFilterSize; j++) - val += lumSrc[j][i] * lumFilter[j]; + val += (lumSrc[j][i] * lumFilter[j]) >> 1; output_pixel(&yDest[i], val); } if (uDest) { for (i = 0; i < chrDstW; i++) { - int u = 1 << (30-output_bits); - int v = 1 << (30-output_bits); + int u = 1 << (30-output_bits - 1); + int v = 1 << (30-output_bits - 1); int j; for (j = 0; j < chrFilterSize; j++) { - u += chrUSrc[j][i] * chrFilter[j]; - v += chrVSrc[j][i] * chrFilter[j]; + u += (chrUSrc[j][i] * chrFilter[j]) >> 1; + v += (chrVSrc[j][i] * chrFilter[j]) >> 1; } output_pixel(&uDest[i], u); @@ -239,11 +239,11 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc, if (CONFIG_SWSCALE_ALPHA && aDest) { for (i = 0; i < dstW; i++) { - int val = 1 << (30-output_bits); + int val = 1 << (30-output_bits - 1); int j; for (j = 0; j < lumFilterSize; j++) - val += alpSrc[j][i] * lumFilter[j]; + val += (alpSrc[j][i] * lumFilter[j]) >> 1; output_pixel(&aDest[i], val); } diff --git a/tests/ref/lavfi/pixdesc b/tests/ref/lavfi/pixdesc index 76d845abb8..a82f8ed9b6 100644 --- a/tests/ref/lavfi/pixdesc +++ b/tests/ref/lavfi/pixdesc @@ -38,12 +38,12 @@ yuv420p16le 2d59c4f1d0314a5a957a7cfc4b6fabcc yuv420p9be ce880fa07830e5297c22acf6e20555ce yuv420p9le 16543fda8f87d94a6cf857d2e8d4461a yuv422p c9bba4529821d796a6ab09f6a5fd355a -yuv422p16be 86ad3447f97969ce095afeef81fa9abf -yuv422p16le a53a9b451f4a81eeae33362c1bbd07f8 +yuv422p16be dc9886f2fccf87cc54b27e071a2c251e +yuv422p16le f181c8d8436f1233ba566d9bc88005ec yuv440p 5a064afe2b453bb52cdb3f176b1aa1cf yuv444p 0a98447b78fd476aa39686da6a74fa2e -yuv444p16be 2a8e2bddfe4c208df4119aaa7dc5db28 -yuv444p16le f2117fc9cf66d3a832183e159ff71803 +yuv444p16be af555dbaa401b142a995566864f47545 +yuv444p16le a803e8016997dad95c5b2a72f54c34d6 yuva420p a29884f3f3dfe1e00b961bc17bef3d47 yuvj420p 32eec78ba51857b16ce9b813a49b7189 yuvj422p 0dfa0ed434f73be51428758c69e082cb diff --git a/tests/ref/lavfi/pixfmts_copy b/tests/ref/lavfi/pixfmts_copy index 76d845abb8..a82f8ed9b6 100644 --- a/tests/ref/lavfi/pixfmts_copy +++ b/tests/ref/lavfi/pixfmts_copy @@ -38,12 +38,12 @@ yuv420p16le 2d59c4f1d0314a5a957a7cfc4b6fabcc yuv420p9be ce880fa07830e5297c22acf6e20555ce yuv420p9le 16543fda8f87d94a6cf857d2e8d4461a yuv422p c9bba4529821d796a6ab09f6a5fd355a -yuv422p16be 86ad3447f97969ce095afeef81fa9abf -yuv422p16le a53a9b451f4a81eeae33362c1bbd07f8 +yuv422p16be dc9886f2fccf87cc54b27e071a2c251e +yuv422p16le f181c8d8436f1233ba566d9bc88005ec yuv440p 5a064afe2b453bb52cdb3f176b1aa1cf yuv444p 0a98447b78fd476aa39686da6a74fa2e -yuv444p16be 2a8e2bddfe4c208df4119aaa7dc5db28 -yuv444p16le f2117fc9cf66d3a832183e159ff71803 +yuv444p16be af555dbaa401b142a995566864f47545 +yuv444p16le a803e8016997dad95c5b2a72f54c34d6 yuva420p a29884f3f3dfe1e00b961bc17bef3d47 yuvj420p 32eec78ba51857b16ce9b813a49b7189 yuvj422p 0dfa0ed434f73be51428758c69e082cb diff --git a/tests/ref/lavfi/pixfmts_crop b/tests/ref/lavfi/pixfmts_crop index fb5c838a4e..e3bb88c101 100644 --- a/tests/ref/lavfi/pixfmts_crop +++ b/tests/ref/lavfi/pixfmts_crop @@ -29,12 +29,12 @@ yuv420p bfea0188ddd4889787c403caae119cc7 yuv420p16be 8365eff38b8c329aeb95fc605fa229bb yuv420p16le 5e8dd38d973d5854abe1ad4efad20cc1 yuv422p f2f930a91fe00d4252c4720b5ecd8961 -yuv422p16be 6647fe1c381c148f8207c988c0e22bf0 -yuv422p16le e1548c9dc51202db38a9625c8954203f +yuv422p16be 93f9b6f33f9529db6de6a9f0ddd70eb5 +yuv422p16le 2e66dcfec54ca6b57aa4bbd9ac234639 yuv440p 2472417d980e395ad6843cbb8b633b29 yuv444p 1f151980486848c96bc5585ced99003e -yuv444p16be 02d78b564a23df2f68cf6895d3bfe6bf -yuv444p16le cbea9591b954ea31d6a0cb25a9aed599 +yuv444p16be e7d1ecf0c11a41b5db192f761f55bd3c +yuv444p16le 3298a0043d982e7cf1a33a1292fa11f0 yuva420p 7536753dfbc7932560fb50c921369a0e yuvj420p 21f891093006d42d7683b0e1d773a657 yuvj422p 9a43d474c407590ad8f213880586b45e diff --git a/tests/ref/lavfi/pixfmts_hflip b/tests/ref/lavfi/pixfmts_hflip index 4a32e35a1c..2084d581e1 100644 --- a/tests/ref/lavfi/pixfmts_hflip +++ b/tests/ref/lavfi/pixfmts_hflip @@ -29,12 +29,12 @@ yuv420p 2d5c80f9ba2ddd85b2aeda3564cc7d64 yuv420p16be 758b0c1e2113b15e7afde48da4e4d024 yuv420p16le 480ccd951dcb806bc875d307e02e50a0 yuv422p 6e728f4eb9eae287c224f396d84be6ea -yuv422p16be 9dbe0af0eb877987611cf04bfa577202 -yuv422p16le 2d8f37231110177cc5e1b61c8cb4b163 +yuv422p16be 8657d2c8d443940300fdb4028d555631 +yuv422p16le 4ab27609981e50de5b1150125718ae76 yuv440p a99e2b57ed601f39852715c9d675d0d3 yuv444p 947e47f7bb5fdccc659d19b7df2b6fc3 -yuv444p16be 38cdb28061ebb8e7aa3797238615e77f -yuv444p16le 80d654986d6f3754e924310a045fdb24 +yuv444p16be a5154ce329db0d2caf0bd43f1347dba3 +yuv444p16le 1f703308b90feb048191b3bccc695671 yuva420p d83ec0c01498189f179ec574918185f1 yuvj420p df3aaaec3bb157c3bde5f0365af30f4f yuvj422p d113871528d510a192797af59df9c05c diff --git a/tests/ref/lavfi/pixfmts_null b/tests/ref/lavfi/pixfmts_null index 76d845abb8..a82f8ed9b6 100644 --- a/tests/ref/lavfi/pixfmts_null +++ b/tests/ref/lavfi/pixfmts_null @@ -38,12 +38,12 @@ yuv420p16le 2d59c4f1d0314a5a957a7cfc4b6fabcc yuv420p9be ce880fa07830e5297c22acf6e20555ce yuv420p9le 16543fda8f87d94a6cf857d2e8d4461a yuv422p c9bba4529821d796a6ab09f6a5fd355a -yuv422p16be 86ad3447f97969ce095afeef81fa9abf -yuv422p16le a53a9b451f4a81eeae33362c1bbd07f8 +yuv422p16be dc9886f2fccf87cc54b27e071a2c251e +yuv422p16le f181c8d8436f1233ba566d9bc88005ec yuv440p 5a064afe2b453bb52cdb3f176b1aa1cf yuv444p 0a98447b78fd476aa39686da6a74fa2e -yuv444p16be 2a8e2bddfe4c208df4119aaa7dc5db28 -yuv444p16le f2117fc9cf66d3a832183e159ff71803 +yuv444p16be af555dbaa401b142a995566864f47545 +yuv444p16le a803e8016997dad95c5b2a72f54c34d6 yuva420p a29884f3f3dfe1e00b961bc17bef3d47 yuvj420p 32eec78ba51857b16ce9b813a49b7189 yuvj422p 0dfa0ed434f73be51428758c69e082cb diff --git a/tests/ref/lavfi/pixfmts_scale b/tests/ref/lavfi/pixfmts_scale index 14c748ff10..094e52462f 100644 --- a/tests/ref/lavfi/pixfmts_scale +++ b/tests/ref/lavfi/pixfmts_scale @@ -31,19 +31,19 @@ uyvy422 314bd486277111a95d9369b944fa0400 yuv410p 7df8f6d69b56a8dcb6c7ee908e5018b5 yuv411p 1143e7c5cc28fe0922b051b17733bc4c yuv420p fdad2d8df8985e3d17e73c71f713cb14 -yuv420p10be aad747a7634ba4ed48f149cdfc78792e -yuv420p10le 46547f19a7f58638dff73657477b11b9 -yuv420p16be d7270efce54eb59c7b01c14157a1b890 -yuv420p16le e85abf00bad940a922b623c91c9026d7 +yuv420p10be af5429f27b9f95bf955e795921c65cdc +yuv420p10le d0b47e6a8a44e6b5ca0fe4349a4e393b +yuv420p16be 9688e33e03b8c8275ab2fb1df0f06bee +yuv420p16le cba8b390ad5e7b8678e419b8ce79c008 yuv420p9be a073b2d93b2a7dce2069ba252bc43175 yuv420p9le b67233c3c7d93763d07d88f697c145e1 yuv422p 918e37701ee7377d16a8a6c119c56a40 -yuv422p16be 5cd8fe1da161d21b65bf75bf4cb50c75 -yuv422p16le 3b545317b1f5e627751525fb2958d88e +yuv422p16be 2cf502d7d386db1f1b3b946679d897b1 +yuv422p16le 3002a4e47520731dcee5929aff49eb74 yuv440p 461503fdb9b90451020aa3b25ddf041c yuv444p 81b2eba962d12e8d64f003ac56f6faf2 -yuv444p16be 2bd3e992d1533f8e0978a8e0c7008243 -yuv444p16le daee5a461ae3bc53295c392e423aa1d7 +yuv444p16be b9f051ce7335923fe33efd162e48da1d +yuv444p16le fa47e317efac988b4a7fa55141c89126 yuva420p 8673a9131fb47de69788863f93a50eb7 yuvj420p 30427bd6caf5bda93a173dbebe759e09 yuvj422p fc8288f64fd149573f73cf8da05d8e6d diff --git a/tests/ref/lavfi/pixfmts_vflip b/tests/ref/lavfi/pixfmts_vflip index b440efa26b..8702eca796 100644 --- a/tests/ref/lavfi/pixfmts_vflip +++ b/tests/ref/lavfi/pixfmts_vflip @@ -38,12 +38,12 @@ yuv420p16le 0f609e588e5a258644ef85170d70e030 yuv420p9be be40ec975fb2873891643cbbbddbc3b0 yuv420p9le 7e606310d3f5ff12badf911e8f333471 yuv422p d7f5cb44d9b0210d66d6a8762640ab34 -yuv422p16be 8cdfbddf2dd4c44c3efef4ee00170eba -yuv422p16le a2f421f6a1af950544081c1797de01ae +yuv422p16be 51d9aa4e78d121c226d919ce97976fe4 +yuv422p16le 12965c54bda8932ca72da194419a9908 yuv440p 876385e96165acf51271b20e5d85a416 yuv444p 9c3c667d1613b72d15bc6d851c5eb8f7 -yuv444p16be bb86de32c67dd49469989ac184b89592 -yuv444p16le 64b899f52d820b14d05ff95954c15790 +yuv444p16be 6502abd75030d462c58d99a8673ec517 +yuv444p16le cd7e88b6d08425450a57555bc86ab210 yuva420p c705d1cf061d8c6580ac690b55f92276 yuvj420p 41fd02b204da0ab62452cd14b595e2e4 yuvj422p 7f6ca9bc1812cde02036d7d29a7cce43