From 1b1b902e2ce19e0276bf847cc87384ec43d230e3 Mon Sep 17 00:00:00 2001 From: Loren Merritt Date: Sat, 22 Sep 2012 01:43:16 +0200 Subject: [PATCH] hqdn3d: Fix out of array read in LOWPASS Fixes ticket1752 Commit message by commiter Signed-off-by: Michael Niedermayer --- libavfilter/vf_hqdn3d.c | 7 +++---- libavfilter/x86/hqdn3d.asm | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/libavfilter/vf_hqdn3d.c b/libavfilter/vf_hqdn3d.c index 51c7e879ab..2fe70563a1 100644 --- a/libavfilter/vf_hqdn3d.c +++ b/libavfilter/vf_hqdn3d.c @@ -50,10 +50,9 @@ void ff_hqdn3d_row_10_x86(uint8_t *src, uint8_t *dst, uint16_t *line_ant, uint16 void ff_hqdn3d_row_16_x86(uint8_t *src, uint8_t *dst, uint16_t *line_ant, uint16_t *frame_ant, ptrdiff_t w, int16_t *spatial, int16_t *temporal); #define LUT_BITS (depth==16 ? 8 : 4) -#define RIGHTSHIFT(a,b) (((a)+(((1<<(b))-1)>>1))>>(b)) -#define LOAD(x) ((depth==8 ? src[x] : AV_RN16A(src+(x)*2)) << (16-depth)) -#define STORE(x,val) (depth==8 ? dst[x] = RIGHTSHIFT(val, 16-depth)\ - : AV_WN16A(dst+(x)*2, RIGHTSHIFT(val, 16-depth))) +#define LOAD(x) (((depth==8 ? src[x] : AV_RN16A(src+(x)*2)) << (16-depth)) + (((1<<(16-depth))-1)>>1)) +#define STORE(x,val) (depth==8 ? dst[x] = (val) >> (16-depth)\ + : AV_WN16A(dst+(x)*2, (val) >> (16-depth))) av_always_inline static uint32_t lowpass(int prev, int cur, int16_t *coef, int depth) diff --git a/libavfilter/x86/hqdn3d.asm b/libavfilter/x86/hqdn3d.asm index 88b9b0d034..d36ce37019 100644 --- a/libavfilter/x86/hqdn3d.asm +++ b/libavfilter/x86/hqdn3d.asm @@ -39,6 +39,7 @@ SECTION .text %endif %if %3 != 16 shl %1, 16-%3 + add %1, (1<<(15-%3))-1 %endif %endmacro @@ -86,7 +87,6 @@ ALIGN 16 mov [frameantq+xq*2], t0w movifnidn dstq, dstmp %if %1 != 16 - add t0d, (1<<(15-%1))-1 shr t0d, 16-%1 ; could eliminate this by storing from t0h, but only with some contraints on register allocation %endif %if %1 == 8