From fccbd1245f391c7ba455f72f7896a6e392d39dfa Mon Sep 17 00:00:00 2001 From: Brian Kim Date: Mon, 13 Jul 2020 10:09:38 -0700 Subject: [PATCH] libavutil/frame: avoid UB when getting plane sizes This uses av_image_fill_plane_sizes instead of av_image_fill_pointers when we are getting plane sizes to avoid UB from adding offsets to NULL. Signed-off-by: Brian Kim Signed-off-by: James Almer --- libavutil/frame.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/libavutil/frame.c b/libavutil/frame.c index 9884eae054..3ab1aa3242 100644 --- a/libavutil/frame.c +++ b/libavutil/frame.c @@ -212,8 +212,10 @@ void av_frame_free(AVFrame **frame) static int get_video_buffer(AVFrame *frame, int align) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format); - int ret, i, padded_height; + int ret, i, padded_height, total_size; int plane_padding = FFMAX(16 + 16/*STRIDE_ALIGN*/, align); + ptrdiff_t linesizes[4]; + size_t sizes[4]; if (!desc) return AVERROR(EINVAL); @@ -238,12 +240,22 @@ static int get_video_buffer(AVFrame *frame, int align) frame->linesize[i] = FFALIGN(frame->linesize[i], align); } + for (i = 0; i < 4; i++) + linesizes[i] = frame->linesize[i]; + padded_height = FFALIGN(frame->height, 32); - if ((ret = av_image_fill_pointers(frame->data, frame->format, padded_height, - NULL, frame->linesize)) < 0) + if ((ret = av_image_fill_plane_sizes(sizes, frame->format, + padded_height, linesizes)) < 0) return ret; - frame->buf[0] = av_buffer_alloc(ret + 4*plane_padding); + total_size = 4*plane_padding; + for (i = 0; i < 4; i++) { + if (sizes[i] > INT_MAX - total_size) + return AVERROR(EINVAL); + total_size += sizes[i]; + } + + frame->buf[0] = av_buffer_alloc(total_size); if (!frame->buf[0]) { ret = AVERROR(ENOMEM); goto fail;