From eb7802afefb7af4da50bc56818cdab9da07de7d0 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 14 Jan 2016 21:33:53 +0100 Subject: [PATCH] swscale/vscale: Check that 2 tap filters are bilinear before using bilinear code Fixes: out of array reads Fixes: 07e8b9c5d348ccdf7add0f37de20cf6c/asan_heap-oob_27e8df7_6849_e56653f768070ec8cb52f587048444c2.mov Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer --- libswscale/swscale_internal.h | 1 + libswscale/vscale.c | 32 +++++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index fa288bda13..1e29ec3227 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -398,6 +398,7 @@ typedef struct SwsContext { uint8_t *chrMmxextFilterCode; ///< Runtime-generated MMXEXT horizontal fast bilinear scaler code for chroma planes. int canMMXEXTBeUsed; + int warned_unuseable_bilinear; int dstY; ///< Last destination vertical line output from last slice. int flags; ///< Flags passed by the user to select scaler algorithm, optimizations, subsampling, etc... diff --git a/libswscale/vscale.c b/libswscale/vscale.c index ff1988f5ed..308a0800e1 100644 --- a/libswscale/vscale.c +++ b/libswscale/vscale.c @@ -26,6 +26,7 @@ typedef struct VScalerContext int filter_size; int isMMX; void *pfn; + yuv2packedX_fn yuv2packedX; } VScalerContext; @@ -123,10 +124,21 @@ static int packed_vscale(SwsContext *c, SwsFilterDescriptor *desc, int sliceY, i uint8_t **dst = desc->dst->plane[0].line + dp; - if (c->yuv2packed1 && lum_fsize == 1 && chr_fsize <= 2) { // unscaled RGB - int chrAlpha = chr_fsize == 1 ? 0 : chr_filter[2 * sliceY + 1]; - ((yuv2packed1_fn)inst->pfn)(c, (const int16_t*)*src0, (const int16_t**)src1, (const int16_t**)src2, (const int16_t*)(desc->alpha ? *src3 : NULL), *dst, dstW, chrAlpha, sliceY); - } else if (c->yuv2packed2 && lum_fsize == 2 && chr_fsize == 2) { // bilinear upscale RGB + if (c->yuv2packed1 && lum_fsize == 1 && chr_fsize == 1) { // unscaled RGB + ((yuv2packed1_fn)inst->pfn)(c, (const int16_t*)*src0, (const int16_t**)src1, (const int16_t**)src2, + (const int16_t*)(desc->alpha ? *src3 : NULL), *dst, dstW, 0, sliceY); + } else if (c->yuv2packed1 && lum_fsize == 1 && chr_fsize == 2 && + chr_filter[2 * sliceY + 1] + chr_filter[2 * chrSliceY] == 4096 && + chr_filter[2 * sliceY + 1] <= 4096U) { // unscaled RGB + int chrAlpha = chr_filter[2 * sliceY + 1]; + ((yuv2packed1_fn)inst->pfn)(c, (const int16_t*)*src0, (const int16_t**)src1, (const int16_t**)src2, + (const int16_t*)(desc->alpha ? *src3 : NULL), *dst, dstW, chrAlpha, sliceY); + } else if (c->yuv2packed2 && lum_fsize == 2 && chr_fsize == 2 && + lum_filter[2 * sliceY + 1] + lum_filter[2 * sliceY] == 4096 && + lum_filter[2 * sliceY + 1] <= 4096U && + chr_filter[2 * chrSliceY + 1] + chr_filter[2 * chrSliceY] == 4096 && + chr_filter[2 * chrSliceY + 1] <= 4096U + ) { // bilinear upscale RGB int lumAlpha = lum_filter[2 * sliceY + 1]; int chrAlpha = chr_filter[2 * sliceY + 1]; c->lumMmxFilter[2] = @@ -136,7 +148,14 @@ static int packed_vscale(SwsContext *c, SwsFilterDescriptor *desc, int sliceY, i ((yuv2packed2_fn)inst->pfn)(c, (const int16_t**)src0, (const int16_t**)src1, (const int16_t**)src2, (const int16_t**)src3, *dst, dstW, lumAlpha, chrAlpha, sliceY); } else { // general RGB - ((yuv2packedX_fn)inst->pfn)(c, lum_filter + sliceY * lum_fsize, + if ((c->yuv2packed1 && lum_fsize == 1 && chr_fsize == 2) || + (c->yuv2packed2 && lum_fsize == 2 && chr_fsize == 2)) { + if (!c->warned_unuseable_bilinear) + av_log(c, AV_LOG_INFO, "Optimized 2 tap filter code cannot be used\n"); + c->warned_unuseable_bilinear = 1; + } + + inst->yuv2packedX(c, lum_filter + sliceY * lum_fsize, (const int16_t**)src0, lum_fsize, chr_filter + sliceY * chr_fsize, (const int16_t**)src1, (const int16_t**)src2, chr_fsize, (const int16_t**)src3, *dst, dstW, sliceY); } @@ -287,8 +306,7 @@ void ff_init_vscale_pfn(SwsContext *c, lumCtx->pfn = yuv2packed1; else if (c->yuv2packed2 && c->vLumFilterSize == 2 && c->vChrFilterSize == 2) lumCtx->pfn = yuv2packed2; - else - lumCtx->pfn = yuv2packedX; + lumCtx->yuv2packedX = yuv2packedX; } else lumCtx->pfn = yuv2anyX; }