From c55478835275240d67c8af9b22de2ecb7e65e444 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 29 Jul 2017 15:55:36 +0200 Subject: [PATCH] avcodec/diracdsp: fix integer overflow Fixes: runtime error: signed integer overflow: 11 * 225726413 cannot be represented in type 'int' Fixes: 2764/clusterfuzz-testcase-minimized-5382561922547712 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer (cherry picked from commit b2d9d7226943d6229a17e31714ce5162bdf88b33) Signed-off-by: Michael Niedermayer --- libavcodec/diracdsp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/diracdsp.c b/libavcodec/diracdsp.c index cd1209e209..8bc79b788c 100644 --- a/libavcodec/diracdsp.c +++ b/libavcodec/diracdsp.c @@ -199,7 +199,7 @@ static void dequant_subband_ ## PX ## _c(uint8_t *src, uint8_t *dst, ptrdiff_t s for (i = 0; i < tot_h; i++) { \ c = *src_r++; \ sign = FFSIGN(c)*(!!c); \ - c = (FFABS(c)*qf + qs) >> 2; \ + c = (FFABS(c)*(unsigned)qf + qs) >> 2; \ *dst_r++ = c*sign; \ } \ src += tot_h << (sizeof(PX) >> 1); \