From 10f1bbfe117e990a6fe56c03085966fa0c0f0f2e Mon Sep 17 00:00:00 2001 From: James Almer Date: Sat, 24 Jun 2023 14:11:57 -0300 Subject: [PATCH] avcodec/j2kenc: use uint64_t type for the lambda value Should fix integer overflows, and improve encoding results. Reviewed-by: Paul B Mahol Signed-off-by: James Almer --- libavcodec/j2kenc.c | 4 ++-- tests/ref/vsynth/vsynth1-jpeg2000-yuva444p16 | 8 ++++---- tests/ref/vsynth/vsynth2-jpeg2000-yuva444p16 | 8 ++++---- tests/ref/vsynth/vsynth3-jpeg2000-yuva444p16 | 8 ++++---- tests/ref/vsynth/vsynth_lena-jpeg2000-yuva444p16 | 8 ++++---- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c index 2b0c58cff4..5f95b772d1 100644 --- a/libavcodec/j2kenc.c +++ b/libavcodec/j2kenc.c @@ -127,7 +127,7 @@ typedef struct { uint8_t *buf_end; int bit_index; - int64_t lambda; + uint64_t lambda; Jpeg2000CodingStyle codsty; Jpeg2000QuantStyle qntsty; @@ -1348,7 +1348,7 @@ static void makelayers(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) } } -static int getcut(Jpeg2000Cblk *cblk, int64_t lambda, int dwt_norm) +static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) { int passno, res = 0; for (passno = 0; passno < cblk->npasses; passno++){ diff --git a/tests/ref/vsynth/vsynth1-jpeg2000-yuva444p16 b/tests/ref/vsynth/vsynth1-jpeg2000-yuva444p16 index 363ddc4ffa..f3d3ba8a6c 100644 --- a/tests/ref/vsynth/vsynth1-jpeg2000-yuva444p16 +++ b/tests/ref/vsynth/vsynth1-jpeg2000-yuva444p16 @@ -1,4 +1,4 @@ -9261706cfd4dfd8db38fc9bdafe6bef7 *tests/data/fate/vsynth1-jpeg2000-yuva444p16.avi -12433462 tests/data/fate/vsynth1-jpeg2000-yuva444p16.avi -6bc7131cd19dbb59339bb13a6d53b94d *tests/data/fate/vsynth1-jpeg2000-yuva444p16.out.rawvideo -stddev: 12.97 PSNR: 25.87 MAXDIFF: 169 bytes: 7603200/ 7603200 +4c051562f5ac7c9871e034917326787c *tests/data/fate/vsynth1-jpeg2000-yuva444p16.avi +12502390 tests/data/fate/vsynth1-jpeg2000-yuva444p16.avi +59d53e883f76d41d17c926236ca9d7d2 *tests/data/fate/vsynth1-jpeg2000-yuva444p16.out.rawvideo +stddev: 2.66 PSNR: 39.62 MAXDIFF: 44 bytes: 7603200/ 7603200 diff --git a/tests/ref/vsynth/vsynth2-jpeg2000-yuva444p16 b/tests/ref/vsynth/vsynth2-jpeg2000-yuva444p16 index bec08da307..3cbe5f4ad2 100644 --- a/tests/ref/vsynth/vsynth2-jpeg2000-yuva444p16 +++ b/tests/ref/vsynth/vsynth2-jpeg2000-yuva444p16 @@ -1,4 +1,4 @@ -54063ab98e48b2a079b63643fe26e082 *tests/data/fate/vsynth2-jpeg2000-yuva444p16.avi -11489842 tests/data/fate/vsynth2-jpeg2000-yuva444p16.avi -a4232b2959472f88b5310572456fa900 *tests/data/fate/vsynth2-jpeg2000-yuva444p16.out.rawvideo -stddev: 11.69 PSNR: 26.77 MAXDIFF: 188 bytes: 7603200/ 7603200 +37555b5b00bba2773d5a4212f6c7debd *tests/data/fate/vsynth2-jpeg2000-yuva444p16.avi +11501630 tests/data/fate/vsynth2-jpeg2000-yuva444p16.avi +61be7920a21190d946c22cb2c945ab6f *tests/data/fate/vsynth2-jpeg2000-yuva444p16.out.rawvideo +stddev: 0.53 PSNR: 53.49 MAXDIFF: 13 bytes: 7603200/ 7603200 diff --git a/tests/ref/vsynth/vsynth3-jpeg2000-yuva444p16 b/tests/ref/vsynth/vsynth3-jpeg2000-yuva444p16 index dae44f40bd..ce92bf7e43 100644 --- a/tests/ref/vsynth/vsynth3-jpeg2000-yuva444p16 +++ b/tests/ref/vsynth/vsynth3-jpeg2000-yuva444p16 @@ -1,4 +1,4 @@ -afdff47e9c71d912153e59a56afdf2a9 *tests/data/fate/vsynth3-jpeg2000-yuva444p16.avi -194250 tests/data/fate/vsynth3-jpeg2000-yuva444p16.avi -9e0ef11f8c85206a950a919cb679f129 *tests/data/fate/vsynth3-jpeg2000-yuva444p16.out.rawvideo -stddev: 10.72 PSNR: 27.53 MAXDIFF: 82 bytes: 86700/ 86700 +ae5951cee3db63c7c6f81d23b601c0fb *tests/data/fate/vsynth3-jpeg2000-yuva444p16.avi +194668 tests/data/fate/vsynth3-jpeg2000-yuva444p16.avi +d959793c1f0bb86b0fc509d5747e03c7 *tests/data/fate/vsynth3-jpeg2000-yuva444p16.out.rawvideo +stddev: 3.06 PSNR: 38.39 MAXDIFF: 40 bytes: 86700/ 86700 diff --git a/tests/ref/vsynth/vsynth_lena-jpeg2000-yuva444p16 b/tests/ref/vsynth/vsynth_lena-jpeg2000-yuva444p16 index d9e856735b..6dc522bb01 100644 --- a/tests/ref/vsynth/vsynth_lena-jpeg2000-yuva444p16 +++ b/tests/ref/vsynth/vsynth_lena-jpeg2000-yuva444p16 @@ -1,4 +1,4 @@ -e9c56e128cf980db7c8e965d2afc0ea6 *tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.avi -11109882 tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.avi -3e1a4f9ca46c2dd3fd997c01a0dfb130 *tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.out.rawvideo -stddev: 11.67 PSNR: 26.79 MAXDIFF: 150 bytes: 7603200/ 7603200 +fcfb2e4738ad6afd7f9d4e9f92084415 *tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.avi +11123818 tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.avi +62ac96a84eb370e5543582512dbe5d7c *tests/data/fate/vsynth_lena-jpeg2000-yuva444p16.out.rawvideo +stddev: 0.45 PSNR: 54.88 MAXDIFF: 7 bytes: 7603200/ 7603200