virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2025-08-04 22:03:09 +02:00
Code Issues Releases Activity

checkasm/h264dsp: Fix stack-buffer-overflow, effective-type violations

Browse Source 
Also ensure that the dst buffers are not too big
(they had the right size for >8 bit depths and were therefore
too big for eight bit, letting potential buffer overflows
in the eight bit version go undetected).

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Andreas Rheinhardt
2025-06-16 12:50:08 +02:00
committed by Michael Niedermayer
parent 3afae0f440
commit 15cec71665
2 changed files with 30 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tests/checkasm/checkasm.h
View File

@ -446,16 +446,18 @@ DECL_CHECKASM_CHECK_FUNC(int32_t);
#define checkasm_check_pixel_padded_align(...) \ #define checkasm_check_pixel_padded_align(...) \
checkasm_check_pixel2(__VA_ARGS__, 8) checkasm_check_pixel2(__VA_ARGS__, 8)
/* This assumes that there is a local variable named "bit_depth". /* This assumes that there is a local variable named "bit_depth"
* and that the type-specific buffers obey the name ## _BITDEPTH
* convention.
* For tests that don't have that and only operate on a single * For tests that don't have that and only operate on a single
* bitdepth, just call checkasm_check(uint8_t, ...) directly. */ * bitdepth, just call checkasm_check(uint8_t, ...) directly. */
#define checkasm_check_dctcoef(buf1, stride1, buf2, stride2, ...) \ #define checkasm_check_dctcoef(buf1, stride1, buf2, stride2, ...) \
((bit_depth > 8) ? \ ((bit_depth > 8) ? \
checkasm_check(int32_t, (const int32_t*)buf1, stride1, \ checkasm_check(int32_t, buf1 ## _32, stride1, \
(const int32_t*)buf2, stride2, \ buf2 ## _32, stride2, \
__VA_ARGS__) : \ __VA_ARGS__) : \
checkasm_check(int16_t, (const int16_t*)buf1, stride1, \ checkasm_check(int16_t, buf1 ## _16, stride1, \
(const int16_t*)buf2, stride2, \ buf2 ## _16, stride2, \
__VA_ARGS__)) __VA_ARGS__))
#endif /* TESTS_CHECKASM_CHECKASM_H */ #endif /* TESTS_CHECKASM_CHECKASM_H */

35
tests/checkasm/h264dsp.c
View File

@ -328,33 +328,44 @@ static void check_idct_multiple(void)
static void check_idct_dequant(void) static void check_idct_dequant(void)
{ {
static const int depths[5] = { 8, 9, 10, 12, 14 }; static const int depths[5] = { 8, 9, 10, 12, 14 };
LOCAL_ALIGNED_16(int16_t, src, [16]); LOCAL_ALIGNED_16(int16_t, src16, [16]);
/* Ensure dst buffers are large enough to hold dctcoefs of all bit-depths. */ LOCAL_ALIGNED_16(int32_t, src32, [16]);
LOCAL_ALIGNED_16(uint8_t, dst0, [16 * 16 * sizeof(int32_t)]); LOCAL_ALIGNED_16(int16_t, dst0_16, [16 * 16]);
LOCAL_ALIGNED_16(uint8_t, dst1, [16 * 16 * sizeof(int32_t)]); LOCAL_ALIGNED_16(int16_t, dst1_16, [16 * 16]);
int16_t *dst_ref = (int16_t *)dst0; LOCAL_ALIGNED_16(int32_t, dst0_32, [16 * 16]);
int16_t *dst_new = (int16_t *)dst1; LOCAL_ALIGNED_16(int32_t, dst1_32, [16 * 16]);
H264DSPContext h; H264DSPContext h;
int bit_depth, i, qmul; int bit_depth, i, qmul;
declare_func_emms(AV_CPU_FLAG_MMX | AV_CPU_FLAG_SSE2, void, int16_t *output, int16_t *input, int qmul); declare_func_emms(AV_CPU_FLAG_MMX | AV_CPU_FLAG_SSE2, void, int16_t *output, int16_t *input, int qmul);
for (int j = 0; j < 16; j++)
src[j] = (rnd() % 512) - 256;
qmul = rnd() % 4096; qmul = rnd() % 4096;
for (i = 0; i < FF_ARRAY_ELEMS(depths); i++) { for (i = 0; i < FF_ARRAY_ELEMS(depths); i++) {
bit_depth = depths[i]; bit_depth = depths[i];
ff_h264dsp_init(&h, bit_depth, 1); ff_h264dsp_init(&h, bit_depth, 1);
memset(dst0, 0, 16 * 16 * SIZEOF_COEF); void *src, *dst_ref, *dst_new;
memset(dst1, 0, 16 * 16 * SIZEOF_COEF); if (bit_depth == 8) {
src = src16;
dst_ref = dst0_16;
dst_new = dst1_16;
for (int j = 0; j < 16; j++)
src16[j] = (rnd() % 512) - 256;
} else {
src = src32;
dst_ref = dst0_32;
dst_new = dst1_32;
for (int j = 0; j < 16; j++)
src32[j] = (rnd() % (1 << (bit_depth + 1))) - (1 << bit_depth);
}
memset(dst_ref, 0, 16 * 16 * SIZEOF_COEF);
memset(dst_new, 0, 16 * 16 * SIZEOF_COEF);
if (check_func(h.h264_luma_dc_dequant_idct, "h264_luma_dc_dequant_idct_%d", bit_depth)) { if (check_func(h.h264_luma_dc_dequant_idct, "h264_luma_dc_dequant_idct_%d", bit_depth)) {
call_ref(dst_ref, src, qmul); call_ref(dst_ref, src, qmul);
call_new(dst_new, src, qmul); call_new(dst_new, src, qmul);
checkasm_check_dctcoef(dst_ref, 16*SIZEOF_COEF, dst_new, 16*SIZEOF_COEF, 16, 16, "dst"); checkasm_check_dctcoef(dst0, 16*SIZEOF_COEF, dst1, 16*SIZEOF_COEF, 16, 16, "dst");
bench_new(dst_new, src, qmul); bench_new(dst_new, src, qmul);
} }
} }