From db54426975e124e98e5130ad01316cb7afd60630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Tue, 29 Mar 2022 15:34:19 +0300 Subject: [PATCH] vc1dsp: Change remaining stride parameters to ptrdiff_t MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing x86 assembly for loop filters uses the stride as a full register without clearing/sign extending the upper half of the registers on x86_64. This avoids crashes if the caller would have passed nonzero bits in the previously undefined upper 32 bits of the parameters. Signed-off-by: Martin Storsjö --- libavcodec/mips/vc1dsp_mips.h | 18 +++++++++--------- libavcodec/mips/vc1dsp_mmi.c | 18 +++++++++--------- libavcodec/vc1dsp.c | 22 +++++++++++----------- libavcodec/vc1dsp.h | 18 +++++++++--------- libavcodec/x86/vc1dsp_init.c | 16 ++++++++-------- libavcodec/x86/vc1dsp_loopfilter.asm | 22 +++++++++++----------- 6 files changed, 57 insertions(+), 57 deletions(-) diff --git a/libavcodec/mips/vc1dsp_mips.h b/libavcodec/mips/vc1dsp_mips.h index 5897daea8c..356153cab6 100644 --- a/libavcodec/mips/vc1dsp_mips.h +++ b/libavcodec/mips/vc1dsp_mips.h @@ -166,17 +166,17 @@ void ff_vc1_inv_trans_4x8_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *blo void ff_vc1_inv_trans_8x4_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block); void ff_vc1_inv_trans_8x8_dc_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block); -void ff_vc1_v_overlap_mmi(uint8_t *src, int stride); -void ff_vc1_h_overlap_mmi(uint8_t *src, int stride); +void ff_vc1_v_overlap_mmi(uint8_t *src, ptrdiff_t stride); +void ff_vc1_h_overlap_mmi(uint8_t *src, ptrdiff_t stride); void ff_vc1_v_s_overlap_mmi(int16_t *top, int16_t *bottom); -void ff_vc1_h_s_overlap_mmi(int16_t *left, int16_t *right, int left_stride, int right_stride, int flags); +void ff_vc1_h_s_overlap_mmi(int16_t *left, int16_t *right, ptrdiff_t left_stride, ptrdiff_t right_stride, int flags); -void ff_vc1_v_loop_filter4_mmi(uint8_t *src, int stride, int pq); -void ff_vc1_h_loop_filter4_mmi(uint8_t *src, int stride, int pq); -void ff_vc1_v_loop_filter8_mmi(uint8_t *src, int stride, int pq); -void ff_vc1_h_loop_filter8_mmi(uint8_t *src, int stride, int pq); -void ff_vc1_v_loop_filter16_mmi(uint8_t *src, int stride, int pq); -void ff_vc1_h_loop_filter16_mmi(uint8_t *src, int stride, int pq); +void ff_vc1_v_loop_filter4_mmi(uint8_t *src, ptrdiff_t stride, int pq); +void ff_vc1_h_loop_filter4_mmi(uint8_t *src, ptrdiff_t stride, int pq); +void ff_vc1_v_loop_filter8_mmi(uint8_t *src, ptrdiff_t stride, int pq); +void ff_vc1_h_loop_filter8_mmi(uint8_t *src, ptrdiff_t stride, int pq); +void ff_vc1_v_loop_filter16_mmi(uint8_t *src, ptrdiff_t stride, int pq); +void ff_vc1_h_loop_filter16_mmi(uint8_t *src, ptrdiff_t stride, int pq); void ff_put_no_rnd_vc1_chroma_mc8_mmi(uint8_t *dst /* align 8 */, uint8_t *src /* align 1 */, diff --git a/libavcodec/mips/vc1dsp_mmi.c b/libavcodec/mips/vc1dsp_mmi.c index 27a3c813da..278aa566bc 100644 --- a/libavcodec/mips/vc1dsp_mmi.c +++ b/libavcodec/mips/vc1dsp_mmi.c @@ -1375,7 +1375,7 @@ void ff_vc1_inv_trans_4x4_mmi(uint8_t *dest, ptrdiff_t linesize, int16_t *block) } /* Apply overlap transform to horizontal edge */ -void ff_vc1_h_overlap_mmi(uint8_t *src, int stride) +void ff_vc1_h_overlap_mmi(uint8_t *src, ptrdiff_t stride) { int i; int a, b, c, d; @@ -1398,7 +1398,7 @@ void ff_vc1_h_overlap_mmi(uint8_t *src, int stride) } } -void ff_vc1_h_s_overlap_mmi(int16_t *left, int16_t *right, int left_stride, int right_stride, int flags) +void ff_vc1_h_s_overlap_mmi(int16_t *left, int16_t *right, ptrdiff_t left_stride, ptrdiff_t right_stride, int flags) { int i; int a, b, c, d; @@ -1428,7 +1428,7 @@ void ff_vc1_h_s_overlap_mmi(int16_t *left, int16_t *right, int left_stride, int } /* Apply overlap transform to vertical edge */ -void ff_vc1_v_overlap_mmi(uint8_t *src, int stride) +void ff_vc1_v_overlap_mmi(uint8_t *src, ptrdiff_t stride) { int i; int a, b, c, d; @@ -1551,32 +1551,32 @@ static inline void vc1_loop_filter(uint8_t *src, int step, int stride, } } -void ff_vc1_v_loop_filter4_mmi(uint8_t *src, int stride, int pq) +void ff_vc1_v_loop_filter4_mmi(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, 1, stride, 4, pq); } -void ff_vc1_h_loop_filter4_mmi(uint8_t *src, int stride, int pq) +void ff_vc1_h_loop_filter4_mmi(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, stride, 1, 4, pq); } -void ff_vc1_v_loop_filter8_mmi(uint8_t *src, int stride, int pq) +void ff_vc1_v_loop_filter8_mmi(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, 1, stride, 8, pq); } -void ff_vc1_h_loop_filter8_mmi(uint8_t *src, int stride, int pq) +void ff_vc1_h_loop_filter8_mmi(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, stride, 1, 8, pq); } -void ff_vc1_v_loop_filter16_mmi(uint8_t *src, int stride, int pq) +void ff_vc1_v_loop_filter16_mmi(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, 1, stride, 16, pq); } -void ff_vc1_h_loop_filter16_mmi(uint8_t *src, int stride, int pq) +void ff_vc1_h_loop_filter16_mmi(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, stride, 1, 16, pq); } diff --git a/libavcodec/vc1dsp.c b/libavcodec/vc1dsp.c index a29b91bf3d..f651d7d461 100644 --- a/libavcodec/vc1dsp.c +++ b/libavcodec/vc1dsp.c @@ -36,7 +36,7 @@ #include "startcode.h" /* Apply overlap transform to horizontal edge */ -static void vc1_v_overlap_c(uint8_t *src, int stride) +static void vc1_v_overlap_c(uint8_t *src, ptrdiff_t stride) { int i; int a, b, c, d; @@ -60,7 +60,7 @@ static void vc1_v_overlap_c(uint8_t *src, int stride) } /* Apply overlap transform to vertical edge */ -static void vc1_h_overlap_c(uint8_t *src, int stride) +static void vc1_h_overlap_c(uint8_t *src, ptrdiff_t stride) { int i; int a, b, c, d; @@ -109,7 +109,7 @@ static void vc1_v_s_overlap_c(int16_t *top, int16_t *bottom) } } -static void vc1_h_s_overlap_c(int16_t *left, int16_t *right, int left_stride, int right_stride, int flags) +static void vc1_h_s_overlap_c(int16_t *left, int16_t *right, ptrdiff_t left_stride, ptrdiff_t right_stride, int flags) { int i; int a, b, c, d; @@ -146,7 +146,7 @@ static void vc1_h_s_overlap_c(int16_t *left, int16_t *right, int left_stride, in * @return whether other 3 pairs should be filtered or not * @see 8.6 */ -static av_always_inline int vc1_filter_line(uint8_t *src, int stride, int pq) +static av_always_inline int vc1_filter_line(uint8_t *src, ptrdiff_t stride, int pq) { int a0 = (2 * (src[-2 * stride] - src[1 * stride]) - 5 * (src[-1 * stride] - src[0 * stride]) + 4) >> 3; @@ -195,7 +195,7 @@ static av_always_inline int vc1_filter_line(uint8_t *src, int stride, int pq) * @param pq block quantizer * @see 8.6 */ -static inline void vc1_loop_filter(uint8_t *src, int step, int stride, +static inline void vc1_loop_filter(uint8_t *src, int step, ptrdiff_t stride, int len, int pq) { int i; @@ -212,32 +212,32 @@ static inline void vc1_loop_filter(uint8_t *src, int step, int stride, } } -static void vc1_v_loop_filter4_c(uint8_t *src, int stride, int pq) +static void vc1_v_loop_filter4_c(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, 1, stride, 4, pq); } -static void vc1_h_loop_filter4_c(uint8_t *src, int stride, int pq) +static void vc1_h_loop_filter4_c(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, stride, 1, 4, pq); } -static void vc1_v_loop_filter8_c(uint8_t *src, int stride, int pq) +static void vc1_v_loop_filter8_c(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, 1, stride, 8, pq); } -static void vc1_h_loop_filter8_c(uint8_t *src, int stride, int pq) +static void vc1_h_loop_filter8_c(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, stride, 1, 8, pq); } -static void vc1_v_loop_filter16_c(uint8_t *src, int stride, int pq) +static void vc1_v_loop_filter16_c(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, 1, stride, 16, pq); } -static void vc1_h_loop_filter16_c(uint8_t *src, int stride, int pq) +static void vc1_h_loop_filter16_c(uint8_t *src, ptrdiff_t stride, int pq) { vc1_loop_filter(src, stride, 1, 16, pq); } diff --git a/libavcodec/vc1dsp.h b/libavcodec/vc1dsp.h index c6443acb20..fe60025a2a 100644 --- a/libavcodec/vc1dsp.h +++ b/libavcodec/vc1dsp.h @@ -42,16 +42,16 @@ typedef struct VC1DSPContext { void (*vc1_inv_trans_8x4_dc)(uint8_t *dest, ptrdiff_t stride, int16_t *block); void (*vc1_inv_trans_4x8_dc)(uint8_t *dest, ptrdiff_t stride, int16_t *block); void (*vc1_inv_trans_4x4_dc)(uint8_t *dest, ptrdiff_t stride, int16_t *block); - void (*vc1_v_overlap)(uint8_t *src, int stride); - void (*vc1_h_overlap)(uint8_t *src, int stride); + void (*vc1_v_overlap)(uint8_t *src, ptrdiff_t stride); + void (*vc1_h_overlap)(uint8_t *src, ptrdiff_t stride); void (*vc1_v_s_overlap)(int16_t *top, int16_t *bottom); - void (*vc1_h_s_overlap)(int16_t *left, int16_t *right, int left_stride, int right_stride, int flags); - void (*vc1_v_loop_filter4)(uint8_t *src, int stride, int pq); - void (*vc1_h_loop_filter4)(uint8_t *src, int stride, int pq); - void (*vc1_v_loop_filter8)(uint8_t *src, int stride, int pq); - void (*vc1_h_loop_filter8)(uint8_t *src, int stride, int pq); - void (*vc1_v_loop_filter16)(uint8_t *src, int stride, int pq); - void (*vc1_h_loop_filter16)(uint8_t *src, int stride, int pq); + void (*vc1_h_s_overlap)(int16_t *left, int16_t *right, ptrdiff_t left_stride, ptrdiff_t right_stride, int flags); + void (*vc1_v_loop_filter4)(uint8_t *src, ptrdiff_t stride, int pq); + void (*vc1_h_loop_filter4)(uint8_t *src, ptrdiff_t stride, int pq); + void (*vc1_v_loop_filter8)(uint8_t *src, ptrdiff_t stride, int pq); + void (*vc1_h_loop_filter8)(uint8_t *src, ptrdiff_t stride, int pq); + void (*vc1_v_loop_filter16)(uint8_t *src, ptrdiff_t stride, int pq); + void (*vc1_h_loop_filter16)(uint8_t *src, ptrdiff_t stride, int pq); /* put 8x8 block with bicubic interpolation and quarterpel precision * last argument is actually round value instead of height diff --git a/libavcodec/x86/vc1dsp_init.c b/libavcodec/x86/vc1dsp_init.c index 8e0c284cc0..2fbf0b3a74 100644 --- a/libavcodec/x86/vc1dsp_init.c +++ b/libavcodec/x86/vc1dsp_init.c @@ -34,18 +34,18 @@ #include "config.h" #define LOOP_FILTER(EXT) \ -void ff_vc1_v_loop_filter4_ ## EXT(uint8_t *src, int stride, int pq); \ -void ff_vc1_h_loop_filter4_ ## EXT(uint8_t *src, int stride, int pq); \ -void ff_vc1_v_loop_filter8_ ## EXT(uint8_t *src, int stride, int pq); \ -void ff_vc1_h_loop_filter8_ ## EXT(uint8_t *src, int stride, int pq); \ +void ff_vc1_v_loop_filter4_ ## EXT(uint8_t *src, ptrdiff_t stride, int pq); \ +void ff_vc1_h_loop_filter4_ ## EXT(uint8_t *src, ptrdiff_t stride, int pq); \ +void ff_vc1_v_loop_filter8_ ## EXT(uint8_t *src, ptrdiff_t stride, int pq); \ +void ff_vc1_h_loop_filter8_ ## EXT(uint8_t *src, ptrdiff_t stride, int pq); \ \ -static void vc1_v_loop_filter16_ ## EXT(uint8_t *src, int stride, int pq) \ +static void vc1_v_loop_filter16_ ## EXT(uint8_t *src, ptrdiff_t stride, int pq) \ { \ ff_vc1_v_loop_filter8_ ## EXT(src, stride, pq); \ ff_vc1_v_loop_filter8_ ## EXT(src+8, stride, pq); \ } \ \ -static void vc1_h_loop_filter16_ ## EXT(uint8_t *src, int stride, int pq) \ +static void vc1_h_loop_filter16_ ## EXT(uint8_t *src, ptrdiff_t stride, int pq) \ { \ ff_vc1_h_loop_filter8_ ## EXT(src, stride, pq); \ ff_vc1_h_loop_filter8_ ## EXT(src+8*stride, stride, pq); \ @@ -56,9 +56,9 @@ LOOP_FILTER(mmxext) LOOP_FILTER(sse2) LOOP_FILTER(ssse3) -void ff_vc1_h_loop_filter8_sse4(uint8_t *src, int stride, int pq); +void ff_vc1_h_loop_filter8_sse4(uint8_t *src, ptrdiff_t stride, int pq); -static void vc1_h_loop_filter16_sse4(uint8_t *src, int stride, int pq) +static void vc1_h_loop_filter16_sse4(uint8_t *src, ptrdiff_t stride, int pq) { ff_vc1_h_loop_filter8_sse4(src, stride, pq); ff_vc1_h_loop_filter8_sse4(src+8*stride, stride, pq); diff --git a/libavcodec/x86/vc1dsp_loopfilter.asm b/libavcodec/x86/vc1dsp_loopfilter.asm index fd33bd13dc..74360949dc 100644 --- a/libavcodec/x86/vc1dsp_loopfilter.asm +++ b/libavcodec/x86/vc1dsp_loopfilter.asm @@ -237,19 +237,19 @@ cglobal vc1_h_loop_filter_internal VC1_H_LOOP_FILTER 4, r4 ret -; void ff_vc1_v_loop_filter4_mmxext(uint8_t *src, int stride, int pq) +; void ff_vc1_v_loop_filter4_mmxext(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_v_loop_filter4, 3,5,0 START_V_FILTER call vc1_v_loop_filter_internal RET -; void ff_vc1_h_loop_filter4_mmxext(uint8_t *src, int stride, int pq) +; void ff_vc1_h_loop_filter4_mmxext(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_h_loop_filter4, 3,5,0 START_H_FILTER 4 call vc1_h_loop_filter_internal RET -; void ff_vc1_v_loop_filter8_mmxext(uint8_t *src, int stride, int pq) +; void ff_vc1_v_loop_filter8_mmxext(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_v_loop_filter8, 3,5,0 START_V_FILTER call vc1_v_loop_filter_internal @@ -258,7 +258,7 @@ cglobal vc1_v_loop_filter8, 3,5,0 call vc1_v_loop_filter_internal RET -; void ff_vc1_h_loop_filter8_mmxext(uint8_t *src, int stride, int pq) +; void ff_vc1_h_loop_filter8_mmxext(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_h_loop_filter8, 3,5,0 START_H_FILTER 4 call vc1_h_loop_filter_internal @@ -271,46 +271,46 @@ INIT_MMX mmxext VC1_LF INIT_XMM sse2 -; void ff_vc1_v_loop_filter8_sse2(uint8_t *src, int stride, int pq) +; void ff_vc1_v_loop_filter8_sse2(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_v_loop_filter8, 3,5,8 START_V_FILTER VC1_V_LOOP_FILTER 8, q RET -; void ff_vc1_h_loop_filter8_sse2(uint8_t *src, int stride, int pq) +; void ff_vc1_h_loop_filter8_sse2(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_h_loop_filter8, 3,6,8 START_H_FILTER 8 VC1_H_LOOP_FILTER 8, r5 RET INIT_MMX ssse3 -; void ff_vc1_v_loop_filter4_ssse3(uint8_t *src, int stride, int pq) +; void ff_vc1_v_loop_filter4_ssse3(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_v_loop_filter4, 3,5,0 START_V_FILTER VC1_V_LOOP_FILTER 4, d RET -; void ff_vc1_h_loop_filter4_ssse3(uint8_t *src, int stride, int pq) +; void ff_vc1_h_loop_filter4_ssse3(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_h_loop_filter4, 3,5,0 START_H_FILTER 4 VC1_H_LOOP_FILTER 4, r4 RET INIT_XMM ssse3 -; void ff_vc1_v_loop_filter8_ssse3(uint8_t *src, int stride, int pq) +; void ff_vc1_v_loop_filter8_ssse3(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_v_loop_filter8, 3,5,8 START_V_FILTER VC1_V_LOOP_FILTER 8, q RET -; void ff_vc1_h_loop_filter8_ssse3(uint8_t *src, int stride, int pq) +; void ff_vc1_h_loop_filter8_ssse3(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_h_loop_filter8, 3,6,8 START_H_FILTER 8 VC1_H_LOOP_FILTER 8, r5 RET INIT_XMM sse4 -; void ff_vc1_h_loop_filter8_sse4(uint8_t *src, int stride, int pq) +; void ff_vc1_h_loop_filter8_sse4(uint8_t *src, ptrdiff_t stride, int pq) cglobal vc1_h_loop_filter8, 3,5,8 START_H_FILTER 8 VC1_H_LOOP_FILTER 8