From 06d174e289eb185f03a34a738965f0042f39c038 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Mon, 26 Apr 2021 22:35:37 +0200 Subject: [PATCH] avformat/rmdec: Check old_format len for overflow Maybe such large values could be disallowed earlier and closer to where they are set. Fixes: signed integer overflow: 538976288 * 8224 cannot be represented in type 'int' Fixes: 29102/clusterfuzz-testcase-minimized-ffmpeg_dem_RM_fuzzer-6704350354341888 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavformat/rmdec.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavformat/rmdec.c b/libavformat/rmdec.c index 68b5c3b75a..ce27bd2299 100644 --- a/libavformat/rmdec.c +++ b/libavformat/rmdec.c @@ -1012,8 +1012,8 @@ static int rm_read_packet(AVFormatContext *s, AVPacket *pkt) { RMDemuxContext *rm = s->priv_data; AVStream *st = NULL; // init to silence compiler warning - int i, len, res, seq = 1; - int64_t timestamp, pos; + int i, res, seq = 1; + int64_t timestamp, pos, len; int flags; for (;;) { @@ -1032,7 +1032,9 @@ static int rm_read_packet(AVFormatContext *s, AVPacket *pkt) ast = st->priv_data; timestamp = AV_NOPTS_VALUE; len = !ast->audio_framesize ? RAW_PACKET_SIZE : - ast->coded_framesize * ast->sub_packet_h / 2; + ast->coded_framesize * (int64_t)ast->sub_packet_h / 2; + if (len > INT_MAX) + return AVERROR_INVALIDDATA; flags = (seq++ == 1) ? 2 : 0; pos = avio_tell(s->pb); } else {