From e229fbf5ce8400fde181758171c6ac911432ddfd Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 21 Jan 2016 15:39:43 +0100 Subject: [PATCH] avcodec/mpegvideo_enc: Check for integer overflow in ff_mpv_reallocate_putbitbuffer() Fixes assertion failure Fixes: 6568d187979ce17878b6fe5fbbb89142/signal_sigabrt_7ffff6ae7cb7_7176_564bbc6741bdcf907f5c4e685c9a77a2.mpg Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer (cherry picked from commit b65efbc0f4195421c15d2a6c228d331eec5b31c3) Signed-off-by: Michael Niedermayer --- libavcodec/mpegvideo_enc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index c4e448251e..b2112c142a 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -2740,6 +2740,11 @@ int ff_mpv_reallocate_putbitbuffer(MpegEncContext *s, size_t threshold, size_t s uint8_t *new_buffer = NULL; int new_buffer_size = 0; + if ((s->avctx->internal->byte_buffer_size + size_increase) >= INT_MAX/8) { + av_log(s->avctx, AV_LOG_ERROR, "Cannot reallocate putbit buffer\n"); + return AVERROR(ENOMEM); + } + av_fast_padded_malloc(&new_buffer, &new_buffer_size, s->avctx->internal->byte_buffer_size + size_increase); if (!new_buffer)