virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2024-11-26 19:01:44 +02:00
Code Issues Releases Activity

avcodec/smcenc: stop accessing out of bounds frame

Browse Source
This commit is contained in:
Paul B Mahol 2022-11-12 15:19:21 +01:00
parent bfab87a61d
commit 13c1310975
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
libavcodec/smcenc.c
View File

@ -61,6 +61,7 @@ typedef struct SMCContext {
{ \ { \
row_ptr += stride * 4; \ row_ptr += stride * 4; \
pixel_ptr = row_ptr; \ pixel_ptr = row_ptr; \
cur_y += 4; \
} \ } \
} \ } \
} }
@ -117,6 +118,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
const uint8_t *prev_pixels = (const uint8_t *)s->prev_frame->data[0]; const uint8_t *prev_pixels = (const uint8_t *)s->prev_frame->data[0];
uint8_t *distinct_values = s->distinct_values; uint8_t *distinct_values = s->distinct_values;
const uint8_t *pixel_ptr, *row_ptr; const uint8_t *pixel_ptr, *row_ptr;
const int height = frame->height;
const int width = frame->width; const int width = frame->width;
uint8_t block_values[16]; uint8_t block_values[16];
int block_counter = 0; int block_counter = 0;
@ -125,13 +127,14 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
int color_octet_index = 0; int color_octet_index = 0;
int color_table_index; /* indexes to color pair, quad, or octet tables */ int color_table_index; /* indexes to color pair, quad, or octet tables */
int total_blocks; int total_blocks;
int cur_y = 0;
memset(s->color_pairs, 0, sizeof(s->color_pairs)); memset(s->color_pairs, 0, sizeof(s->color_pairs));
memset(s->color_quads, 0, sizeof(s->color_quads)); memset(s->color_quads, 0, sizeof(s->color_quads));
memset(s->color_octets, 0, sizeof(s->color_octets)); memset(s->color_octets, 0, sizeof(s->color_octets));
/* Number of 4x4 blocks in frame. */ /* Number of 4x4 blocks in frame. */
total_blocks = ((frame->width + 3) / 4) * ((frame->height + 3) / 4); total_blocks = ((width + 3) / 4) * ((height + 3) / 4);
pixel_ptr = row_ptr = src_pixels; pixel_ptr = row_ptr = src_pixels;
@ -145,11 +148,13 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
int cache_index; int cache_index;
int distinct = 0; int distinct = 0;
int blocks = 0; int blocks = 0;
int frame_y = cur_y;
while (prev_pixels && s->key_frame == 0 && block_counter + inter_skip_blocks < total_blocks) { while (prev_pixels && s->key_frame == 0 && block_counter + inter_skip_blocks < total_blocks) {
const int y_size = FFMIN(4, height - cur_y);
int compare = 0; int compare = 0;
for (int y = 0; y < 4; y++) { for (int y = 0; y < y_size; y++) {
const ptrdiff_t offset = pixel_ptr - src_pixels; const ptrdiff_t offset = pixel_ptr - src_pixels;
const uint8_t *prev_pixel_ptr = prev_pixels + offset; const uint8_t *prev_pixel_ptr = prev_pixels + offset;
@ -170,8 +175,10 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
pixel_ptr = xpixel_ptr; pixel_ptr = xpixel_ptr;
row_ptr = xrow_ptr; row_ptr = xrow_ptr;
cur_y = frame_y;
while (block_counter > 0 && block_counter + intra_skip_blocks < total_blocks) { while (block_counter > 0 && block_counter + intra_skip_blocks < total_blocks) {
const int y_size = FFMIN(4, height - cur_y);
const ptrdiff_t offset = pixel_ptr - src_pixels; const ptrdiff_t offset = pixel_ptr - src_pixels;
const int sy = offset / stride; const int sy = offset / stride;
const int sx = offset % stride; const int sx = offset % stride;
@ -180,7 +187,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
const uint8_t *old_pixel_ptr = src_pixels + nx + ny * stride; const uint8_t *old_pixel_ptr = src_pixels + nx + ny * stride;
int compare = 0; int compare = 0;
for (int y = 0; y < 4; y++) { for (int y = 0; y < y_size; y++) {
compare |= memcmp(old_pixel_ptr + y * stride, pixel_ptr + y * stride, 4); compare |= memcmp(old_pixel_ptr + y * stride, pixel_ptr + y * stride, 4);
if (compare) if (compare)
break; break;
@ -197,9 +204,11 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
pixel_ptr = xpixel_ptr; pixel_ptr = xpixel_ptr;
row_ptr = xrow_ptr; row_ptr = xrow_ptr;
cur_y = frame_y;
while (block_counter + coded_blocks < total_blocks && coded_blocks < 256) { while (block_counter + coded_blocks < total_blocks && coded_blocks < 256) {
for (int y = 0; y < 4; y++) const int y_size = FFMIN(4, height - cur_y);
for (int y = 0; y < y_size; y++)
memcpy(block_values + y * 4, pixel_ptr + y * stride, 4); memcpy(block_values + y * 4, pixel_ptr + y * stride, 4);
qsort(block_values, 16, sizeof(block_values[0]), smc_cmp_values); qsort(block_values, 16, sizeof(block_values[0]), smc_cmp_values);
@ -224,6 +233,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
pixel_ptr = xpixel_ptr; pixel_ptr = xpixel_ptr;
row_ptr = xrow_ptr; row_ptr = xrow_ptr;
cur_y = frame_y;
blocks = coded_blocks; blocks = coded_blocks;
distinct = coded_distinct; distinct = coded_distinct;