From fe4c76b3d09fb1d93efd438fac35ef16324d40cb Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 8 Jul 2014 17:05:27 +0200 Subject: [PATCH] avcodec/roqvideoenc: fix infinite lambda increasing loop The threshold was choosen so that no further size decrease happened with larger lambda with the test video. Signed-off-by: Michael Niedermayer --- libavcodec/roqvideoenc.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/libavcodec/roqvideoenc.c b/libavcodec/roqvideoenc.c index df15196e10..6530ec7d38 100644 --- a/libavcodec/roqvideoenc.c +++ b/libavcodec/roqvideoenc.c @@ -881,7 +881,7 @@ static void generate_new_codebooks(RoqContext *enc, RoqTempdata *tempData) av_free(results4); } -static void roq_encode_video(RoqContext *enc) +static int roq_encode_video(RoqContext *enc) { RoqTempdata *tempData = enc->tmpData; int i; @@ -903,6 +903,10 @@ static void roq_encode_video(RoqContext *enc) /* Quake 3 can't handle chunks bigger than 65535 bytes */ if (tempData->mainChunkSize/8 > 65535 && enc->quake3_compat) { + if (enc->lambda > 100000) { + av_log(enc->avctx, AV_LOG_ERROR, "Cannot encode video in Quake compatible form\n"); + return AVERROR(EINVAL); + } av_log(enc->avctx, AV_LOG_ERROR, "Warning, generated a frame too big for Quake (%d > 65535), " "now switching to a bigger qscale value.\n", @@ -936,6 +940,8 @@ static void roq_encode_video(RoqContext *enc) av_free(tempData->closest_cb2); enc->framesSinceKeyframe++; + + return 0; } static av_cold int roq_encode_end(AVCodecContext *avctx) @@ -1064,7 +1070,8 @@ static int roq_encode_frame(AVCodecContext *avctx, AVPacket *pkt, } /* Encode the actual frame */ - roq_encode_video(enc); + if ((ret = roq_encode_video(enc)) < 0) + return ret; pkt->size = enc->out_buf - pkt->data; if (enc->framesSinceKeyframe == 1)