virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2025-01-24 13:56:33 +02:00
Code Issues Releases Activity

avcodec/simple_idct_template: Fix several integer overflows

Browse Source 
Benchmarks with START_TIMER indicate that the code is faster with unsigned, (that is
with the patch), there was quite some fluctuation in the numbers so this may be just
random

Fixes: 811/clusterfuzz-testcase-6465493076541440

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2017-03-13 02:25:42 +01:00
parent b409d8d4a2
commit 45198477de
1 changed files with 18 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
libavcodec/simple_idct_template.c
View File

@ -112,7 +112,7 @@ static inline void FUNC(idctRowCondDC_extrashift)(int16_t *row, int extra_shift)
static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift) static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift)
#endif #endif
{ {
int a0, a1, a2, a3, b0, b1, b2, b3; SUINT a0, a1, a2, a3, b0, b1, b2, b3;
#if HAVE_FAST_64BIT #if HAVE_FAST_64BIT
#define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN) #define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN)
@ -187,14 +187,14 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift)
MAC(b3, -W1, row[7]); MAC(b3, -W1, row[7]);
} }
row[0] = (a0 + b0) >> (ROW_SHIFT + extra_shift); row[0] = (int)(a0 + b0) >> (ROW_SHIFT + extra_shift);
row[7] = (a0 - b0) >> (ROW_SHIFT + extra_shift); row[7] = (int)(a0 - b0) >> (ROW_SHIFT + extra_shift);
row[1] = (a1 + b1) >> (ROW_SHIFT + extra_shift); row[1] = (int)(a1 + b1) >> (ROW_SHIFT + extra_shift);
row[6] = (a1 - b1) >> (ROW_SHIFT + extra_shift); row[6] = (int)(a1 - b1) >> (ROW_SHIFT + extra_shift);
row[2] = (a2 + b2) >> (ROW_SHIFT + extra_shift); row[2] = (int)(a2 + b2) >> (ROW_SHIFT + extra_shift);
row[5] = (a2 - b2) >> (ROW_SHIFT + extra_shift); row[5] = (int)(a2 - b2) >> (ROW_SHIFT + extra_shift);
row[3] = (a3 + b3) >> (ROW_SHIFT + extra_shift); row[3] = (int)(a3 + b3) >> (ROW_SHIFT + extra_shift);
row[4] = (a3 - b3) >> (ROW_SHIFT + extra_shift); row[4] = (int)(a3 - b3) >> (ROW_SHIFT + extra_shift);
} }
#define IDCT_COLS do { \ #define IDCT_COLS do { \
@ -253,25 +253,25 @@ static inline void FUNC(idctSparseCol_extrashift)(int16_t *col)
static inline void FUNC(idctSparseColPut)(pixel *dest, int line_size, static inline void FUNC(idctSparseColPut)(pixel *dest, int line_size,
int16_t *col) int16_t *col)
{ {
int a0, a1, a2, a3, b0, b1, b2, b3; SUINT a0, a1, a2, a3, b0, b1, b2, b3;
IDCT_COLS; IDCT_COLS;
dest[0] = av_clip_pixel((a0 + b0) >> COL_SHIFT); dest[0] = av_clip_pixel((int)(a0 + b0) >> COL_SHIFT);
dest += line_size; dest += line_size;
dest[0] = av_clip_pixel((a1 + b1) >> COL_SHIFT); dest[0] = av_clip_pixel((int)(a1 + b1) >> COL_SHIFT);
dest += line_size; dest += line_size;
dest[0] = av_clip_pixel((a2 + b2) >> COL_SHIFT); dest[0] = av_clip_pixel((int)(a2 + b2) >> COL_SHIFT);
dest += line_size; dest += line_size;
dest[0] = av_clip_pixel((a3 + b3) >> COL_SHIFT); dest[0] = av_clip_pixel((int)(a3 + b3) >> COL_SHIFT);
dest += line_size; dest += line_size;
dest[0] = av_clip_pixel((a3 - b3) >> COL_SHIFT); dest[0] = av_clip_pixel((int)(a3 - b3) >> COL_SHIFT);
dest += line_size; dest += line_size;
dest[0] = av_clip_pixel((a2 - b2) >> COL_SHIFT); dest[0] = av_clip_pixel((int)(a2 - b2) >> COL_SHIFT);
dest += line_size; dest += line_size;
dest[0] = av_clip_pixel((a1 - b1) >> COL_SHIFT); dest[0] = av_clip_pixel((int)(a1 - b1) >> COL_SHIFT);
dest += line_size; dest += line_size;
dest[0] = av_clip_pixel((a0 - b0) >> COL_SHIFT); dest[0] = av_clip_pixel((int)(a0 - b0) >> COL_SHIFT);
} }
static inline void FUNC(idctSparseColAdd)(pixel *dest, int line_size, static inline void FUNC(idctSparseColAdd)(pixel *dest, int line_size,