mirror of
https://github.com/FFmpeg/FFmpeg.git
synced 2024-11-26 19:01:44 +02:00
huffyuv: Check and propagate function return values
Bug-Id: CVE-2013-0868 inspired by a patch from Michael Niedermayer <michaelni@gmx.at> Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Diego Biurrun <diego@biurrun.de> CC: libav-stable@libav.org
This commit is contained in:
Diego Biurrun
parent
6234058148
commit
d0393d79bc
@ -106,11 +106,13 @@ static int read_len_table(uint8_t *dst, GetBitContext *gb)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void generate_joint_tables(HYuvContext *s)
|
||||
static int generate_joint_tables(HYuvContext *s)
|
||||
{
|
||||
uint16_t symbols[1 << VLC_BITS];
|
||||
uint16_t bits[1 << VLC_BITS];
|
||||
uint8_t len[1 << VLC_BITS];
|
||||
int ret;
|
||||
|
||||
if (s->bitstream_bpp < 24) {
|
||||
int p, i, y, u;
|
||||
for (p = 0; p < 3; p++) {
|
||||
@ -131,8 +133,9 @@ static void generate_joint_tables(HYuvContext *s)
|
||||
}
|
||||
}
|
||||
ff_free_vlc(&s->vlc[3 + p]);
|
||||
ff_init_vlc_sparse(&s->vlc[3 + p], VLC_BITS, i, len, 1, 1,
|
||||
bits, 2, 2, symbols, 2, 2, 0);
|
||||
if ((ret = ff_init_vlc_sparse(&s->vlc[3 + p], VLC_BITS, i, len, 1, 1,
|
||||
bits, 2, 2, symbols, 2, 2, 0)) < 0)
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
uint8_t (*map)[4] = (uint8_t(*)[4]) s->pix_bgr_map;
|
||||
@ -173,28 +176,34 @@ static void generate_joint_tables(HYuvContext *s)
|
||||
}
|
||||
}
|
||||
ff_free_vlc(&s->vlc[3]);
|
||||
init_vlc(&s->vlc[3], VLC_BITS, i, len, 1, 1, bits, 2, 2, 0);
|
||||
if ((ret = init_vlc(&s->vlc[3], VLC_BITS, i, len, 1, 1,
|
||||
bits, 2, 2, 0)) < 0)
|
||||
return ret;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length)
|
||||
{
|
||||
GetBitContext gb;
|
||||
int i;
|
||||
int i, ret;
|
||||
|
||||
init_get_bits(&gb, src, length * 8);
|
||||
if ((ret = init_get_bits(&gb, src, length * 8)) < 0)
|
||||
return ret;
|
||||
|
||||
for (i = 0; i < 3; i++) {
|
||||
if (read_len_table(s->len[i], &gb) < 0)
|
||||
return -1;
|
||||
if (ff_huffyuv_generate_bits_table(s->bits[i], s->len[i]) < 0)
|
||||
return -1;
|
||||
if ((ret = read_len_table(s->len[i], &gb)) < 0)
|
||||
return ret;
|
||||
if ((ret = ff_huffyuv_generate_bits_table(s->bits[i], s->len[i])) < 0)
|
||||
return ret;
|
||||
ff_free_vlc(&s->vlc[i]);
|
||||
init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
|
||||
s->bits[i], 4, 4, 0);
|
||||
if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
|
||||
s->bits[i], 4, 4, 0)) < 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
generate_joint_tables(s);
|
||||
if ((ret = generate_joint_tables(s)) < 0)
|
||||
return ret;
|
||||
|
||||
return (get_bits_count(&gb) + 7) / 8;
|
||||
}
|
||||
@ -202,17 +211,19 @@ static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length)
|
||||
static int read_old_huffman_tables(HYuvContext *s)
|
||||
{
|
||||
GetBitContext gb;
|
||||
int i;
|
||||
int i, ret;
|
||||
|
||||
init_get_bits(&gb, classic_shift_luma,
|
||||
classic_shift_luma_table_size * 8);
|
||||
if (read_len_table(s->len[0], &gb) < 0)
|
||||
return -1;
|
||||
if ((ret = init_get_bits(&gb, classic_shift_luma,
|
||||
classic_shift_luma_table_size * 8)) < 0)
|
||||
return ret;
|
||||
if ((ret = read_len_table(s->len[0], &gb)) < 0)
|
||||
return ret;
|
||||
|
||||
init_get_bits(&gb, classic_shift_chroma,
|
||||
classic_shift_chroma_table_size * 8);
|
||||
if (read_len_table(s->len[1], &gb) < 0)
|
||||
return -1;
|
||||
if ((ret = init_get_bits(&gb, classic_shift_chroma,
|
||||
classic_shift_chroma_table_size * 8)) < 0)
|
||||
return ret;
|
||||
if ((ret = read_len_table(s->len[1], &gb)) < 0)
|
||||
return ret;
|
||||
|
||||
for (i = 0; i < 256; i++)
|
||||
s->bits[0][i] = classic_add_luma[i];
|
||||
@ -228,11 +239,13 @@ static int read_old_huffman_tables(HYuvContext *s)
|
||||
|
||||
for (i = 0; i < 3; i++) {
|
||||
ff_free_vlc(&s->vlc[i]);
|
||||
init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
|
||||
s->bits[i], 4, 4, 0);
|
||||
if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
|
||||
s->bits[i], 4, 4, 0)) < 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
generate_joint_tables(s);
|
||||
if ((ret = generate_joint_tables(s)) < 0)
|
||||
return ret;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -240,6 +253,7 @@ static int read_old_huffman_tables(HYuvContext *s)
|
||||
static av_cold int decode_init(AVCodecContext *avctx)
|
||||
{
|
||||
HYuvContext *s = avctx->priv_data;
|
||||
int ret;
|
||||
|
||||
ff_huffyuv_common_init(avctx);
|
||||
ff_huffyuvdsp_init(&s->hdsp);
|
||||
@ -273,9 +287,9 @@ static av_cold int decode_init(AVCodecContext *avctx)
|
||||
s->interlaced = (interlace == 1) ? 1 : (interlace == 2) ? 0 : s->interlaced;
|
||||
s->context = avctx->extradata[2] & 0x40 ? 1 : 0;
|
||||
|
||||
if (read_huffman_tables(s, avctx->extradata + 4,
|
||||
avctx->extradata_size - 4) < 0)
|
||||
return -1;
|
||||
if ((ret = read_huffman_tables(s, avctx->extradata + 4,
|
||||
avctx->extradata_size - 4)) < 0)
|
||||
return ret;
|
||||
} else {
|
||||
switch (avctx->bits_per_coded_sample & 7) {
|
||||
case 1:
|
||||
@ -302,8 +316,8 @@ static av_cold int decode_init(AVCodecContext *avctx)
|
||||
s->bitstream_bpp = avctx->bits_per_coded_sample & ~7;
|
||||
s->context = 0;
|
||||
|
||||
if (read_old_huffman_tables(s) < 0)
|
||||
return -1;
|
||||
if ((ret = read_old_huffman_tables(s)) < 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
switch (s->bitstream_bpp) {
|
||||
@ -327,7 +341,8 @@ static av_cold int decode_init(AVCodecContext *avctx)
|
||||
return AVERROR_INVALIDDATA;
|
||||
}
|
||||
|
||||
ff_huffyuv_alloc_temp(s);
|
||||
if ((ret = ff_huffyuv_alloc_temp(s)) < 0)
|
||||
return ret;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -335,20 +350,21 @@ static av_cold int decode_init(AVCodecContext *avctx)
|
||||
static av_cold int decode_init_thread_copy(AVCodecContext *avctx)
|
||||
{
|
||||
HYuvContext *s = avctx->priv_data;
|
||||
int i;
|
||||
int i, ret;
|
||||
|
||||
ff_huffyuv_alloc_temp(s);
|
||||
if ((ret = ff_huffyuv_alloc_temp(s)) < 0)
|
||||
return ret;
|
||||
|
||||
for (i = 0; i < 6; i++)
|
||||
s->vlc[i].table = NULL;
|
||||
|
||||
if (s->version == 2) {
|
||||
if (read_huffman_tables(s, avctx->extradata + 4,
|
||||
avctx->extradata_size) < 0)
|
||||
return -1;
|
||||
if ((ret = read_huffman_tables(s, avctx->extradata + 4,
|
||||
avctx->extradata_size)) < 0)
|
||||
return ret;
|
||||
} else {
|
||||
if (read_old_huffman_tables(s) < 0)
|
||||
return -1;
|
||||
if ((ret = read_old_huffman_tables(s)) < 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
@ -484,7 +500,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
|
||||
int fake_ystride, fake_ustride, fake_vstride;
|
||||
ThreadFrame frame = { .f = data };
|
||||
AVFrame *const p = data;
|
||||
int table_size = 0;
|
||||
int table_size = 0, ret;
|
||||
|
||||
av_fast_malloc(&s->bitstream_buffer,
|
||||
&s->bitstream_buffer_size,
|
||||
@ -496,22 +512,23 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
|
||||
s->bdsp.bswap_buf((uint32_t *) s->bitstream_buffer,
|
||||
(const uint32_t *) buf, buf_size / 4);
|
||||
|
||||
if (ff_thread_get_buffer(avctx, &frame, 0) < 0) {
|
||||
if ((ret = ff_thread_get_buffer(avctx, &frame, 0)) < 0) {
|
||||
av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
|
||||
return -1;
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (s->context) {
|
||||
table_size = read_huffman_tables(s, s->bitstream_buffer, buf_size);
|
||||
if (table_size < 0)
|
||||
return -1;
|
||||
return table_size;
|
||||
}
|
||||
|
||||
if ((unsigned) (buf_size - table_size) >= INT_MAX / 8)
|
||||
return AVERROR_INVALIDDATA;
|
||||
|
||||
init_get_bits(&s->gb, s->bitstream_buffer + table_size,
|
||||
(buf_size - table_size) * 8);
|
||||
if ((ret = init_get_bits(&s->gb, s->bitstream_buffer + table_size,
|
||||
(buf_size - table_size) * 8)) < 0)
|
||||
return ret;
|
||||
|
||||
fake_ystride = s->interlaced ? p->linesize[0] * 2 : p->linesize[0];
|
||||
fake_ustride = s->interlaced ? p->linesize[1] * 2 : p->linesize[1];
|
||||
|
||||
Loading…
Reference in New Issue
Block a user