virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2025-01-24 13:56:33 +02:00
Code Issues Releases Activity

swscale/vscale: Check that 2 tap filters are bilinear before using bilinear code

Browse Source 
Fixes: out of array reads
Fixes: 07e8b9c5d348ccdf7add0f37de20cf6c/asan_heap-oob_27e8df7_6849_e56653f768070ec8cb52f587048444c2.mov

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2016-01-14 21:33:53 +01:00
parent b99f498716
commit eb7802afef
2 changed files with 26 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
libswscale/swscale_internal.h
View File

@ -398,6 +398,7 @@ typedef struct SwsContext {
uint8_t *chrMmxextFilterCode; ///< Runtime-generated MMXEXT horizontal fast bilinear scaler code for chroma planes. uint8_t *chrMmxextFilterCode; ///< Runtime-generated MMXEXT horizontal fast bilinear scaler code for chroma planes.
int canMMXEXTBeUsed; int canMMXEXTBeUsed;
int warned_unuseable_bilinear;
int dstY; ///< Last destination vertical line output from last slice. int dstY; ///< Last destination vertical line output from last slice.
int flags; ///< Flags passed by the user to select scaler algorithm, optimizations, subsampling, etc... int flags; ///< Flags passed by the user to select scaler algorithm, optimizations, subsampling, etc...

32
libswscale/vscale.c
View File

@ -26,6 +26,7 @@ typedef struct VScalerContext
int filter_size; int filter_size;
int isMMX; int isMMX;
void *pfn; void *pfn;
yuv2packedX_fn yuv2packedX;
} VScalerContext; } VScalerContext;
@ -123,10 +124,21 @@ static int packed_vscale(SwsContext *c, SwsFilterDescriptor *desc, int sliceY, i
uint8_t **dst = desc->dst->plane[0].line + dp; uint8_t **dst = desc->dst->plane[0].line + dp;
if (c->yuv2packed1 && lum_fsize == 1 && chr_fsize <= 2) { // unscaled RGB if (c->yuv2packed1 && lum_fsize == 1 && chr_fsize == 1) { // unscaled RGB
int chrAlpha = chr_fsize == 1 ? 0 : chr_filter[2 * sliceY + 1]; ((yuv2packed1_fn)inst->pfn)(c, (const int16_t*)*src0, (const int16_t**)src1, (const int16_t**)src2,
((yuv2packed1_fn)inst->pfn)(c, (const int16_t*)*src0, (const int16_t**)src1, (const int16_t**)src2, (const int16_t*)(desc->alpha ? *src3 : NULL), *dst, dstW, chrAlpha, sliceY); (const int16_t*)(desc->alpha ? *src3 : NULL), *dst, dstW, 0, sliceY);
} else if (c->yuv2packed2 && lum_fsize == 2 && chr_fsize == 2) { // bilinear upscale RGB } else if (c->yuv2packed1 && lum_fsize == 1 && chr_fsize == 2 &&
chr_filter[2 * sliceY + 1] + chr_filter[2 * chrSliceY] == 4096 &&
chr_filter[2 * sliceY + 1] <= 4096U) { // unscaled RGB
int chrAlpha = chr_filter[2 * sliceY + 1];
((yuv2packed1_fn)inst->pfn)(c, (const int16_t*)*src0, (const int16_t**)src1, (const int16_t**)src2,
(const int16_t*)(desc->alpha ? *src3 : NULL), *dst, dstW, chrAlpha, sliceY);
} else if (c->yuv2packed2 && lum_fsize == 2 && chr_fsize == 2 &&
lum_filter[2 * sliceY + 1] + lum_filter[2 * sliceY] == 4096 &&
lum_filter[2 * sliceY + 1] <= 4096U &&
chr_filter[2 * chrSliceY + 1] + chr_filter[2 * chrSliceY] == 4096 &&
chr_filter[2 * chrSliceY + 1] <= 4096U
) { // bilinear upscale RGB
int lumAlpha = lum_filter[2 * sliceY + 1]; int lumAlpha = lum_filter[2 * sliceY + 1];
int chrAlpha = chr_filter[2 * sliceY + 1]; int chrAlpha = chr_filter[2 * sliceY + 1];
c->lumMmxFilter[2] = c->lumMmxFilter[2] =
@ -136,7 +148,14 @@ static int packed_vscale(SwsContext *c, SwsFilterDescriptor *desc, int sliceY, i
((yuv2packed2_fn)inst->pfn)(c, (const int16_t**)src0, (const int16_t**)src1, (const int16_t**)src2, (const int16_t**)src3, ((yuv2packed2_fn)inst->pfn)(c, (const int16_t**)src0, (const int16_t**)src1, (const int16_t**)src2, (const int16_t**)src3,
*dst, dstW, lumAlpha, chrAlpha, sliceY); *dst, dstW, lumAlpha, chrAlpha, sliceY);
} else { // general RGB } else { // general RGB
((yuv2packedX_fn)inst->pfn)(c, lum_filter + sliceY * lum_fsize, if ((c->yuv2packed1 && lum_fsize == 1 && chr_fsize == 2) ||
(c->yuv2packed2 && lum_fsize == 2 && chr_fsize == 2)) {
if (!c->warned_unuseable_bilinear)
av_log(c, AV_LOG_INFO, "Optimized 2 tap filter code cannot be used\n");
c->warned_unuseable_bilinear = 1;
}
inst->yuv2packedX(c, lum_filter + sliceY * lum_fsize,
(const int16_t**)src0, lum_fsize, chr_filter + sliceY * chr_fsize, (const int16_t**)src0, lum_fsize, chr_filter + sliceY * chr_fsize,
(const int16_t**)src1, (const int16_t**)src2, chr_fsize, (const int16_t**)src3, *dst, dstW, sliceY); (const int16_t**)src1, (const int16_t**)src2, chr_fsize, (const int16_t**)src3, *dst, dstW, sliceY);
} }
@ -287,8 +306,7 @@ void ff_init_vscale_pfn(SwsContext *c,
lumCtx->pfn = yuv2packed1; lumCtx->pfn = yuv2packed1;
else if (c->yuv2packed2 && c->vLumFilterSize == 2 && c->vChrFilterSize == 2) else if (c->yuv2packed2 && c->vLumFilterSize == 2 && c->vChrFilterSize == 2)
lumCtx->pfn = yuv2packed2; lumCtx->pfn = yuv2packed2;
else lumCtx->yuv2packedX = yuv2packedX;
lumCtx->pfn = yuv2packedX;
} else } else
lumCtx->pfn = yuv2anyX; lumCtx->pfn = yuv2anyX;
} }