From aae478036223ae16c24b9890d087feda2efbe38a Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 1 Dec 2012 22:09:14 +0100 Subject: [PATCH] vmnc: Check for integer overflow Fixes null pointer dereference and potential out of array accesses. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer --- libavcodec/vmnc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/vmnc.c b/libavcodec/vmnc.c index d3c86f1f97..d465f305cc 100644 --- a/libavcodec/vmnc.c +++ b/libavcodec/vmnc.c @@ -345,6 +345,10 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac size_left = buf_size - (src - buf); switch(enc) { case MAGIC_WMVd: // cursor + if (w*(int64_t)h*c->bpp2 > INT_MAX/2 - 2) { + av_log(avctx, AV_LOG_ERROR, "dimensions too large\n"); + return AVERROR_INVALIDDATA; + } if(size_left < 2 + w * h * c->bpp2 * 2) { av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left); return -1;