virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2024-12-23 12:43:46 +02:00
Code Issues Releases Activity

xxan: convert to bytestream2 API.

Browse Source 
Protects against overreads.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
This commit is contained in:
Ronald S. Bultje 2012-03-08 16:32:47 -08:00
parent f1279e286b
commit 5518827816
1 changed files with 56 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
libavcodec/xxan.c
View File

@ -35,6 +35,7 @@ typedef struct XanContext {
uint8_t *y_buffer; uint8_t *y_buffer;
uint8_t *scratch_buffer; uint8_t *scratch_buffer;
int buffer_size; int buffer_size;
GetByteContext gb;
} XanContext; } XanContext;
static av_cold int xan_decode_init(AVCodecContext *avctx) static av_cold int xan_decode_init(AVCodecContext *avctx)
@ -58,29 +59,29 @@ static av_cold int xan_decode_init(AVCodecContext *avctx)
return 0; return 0;
} }
static int xan_unpack_luma(const uint8_t *src, const int src_size, static int xan_unpack_luma(XanContext *s,
uint8_t *dst, const int dst_size) uint8_t *dst, const int dst_size)
{ {
int tree_size, eof; int tree_size, eof;
const uint8_t *tree;
int bits, mask; int bits, mask;
int tree_root, node; int tree_root, node;
const uint8_t *dst_end = dst + dst_size; const uint8_t *dst_end = dst + dst_size;
const uint8_t *src_end = src + src_size; GetByteContext tree = s->gb;
int start_off = bytestream2_tell(&tree);
tree_size = *src++; tree_size = bytestream2_get_byte(&s->gb);
eof = *src++; eof = bytestream2_get_byte(&s->gb);
tree = src - eof * 2 - 2;
tree_root = eof + tree_size; tree_root = eof + tree_size;
src += tree_size * 2; bytestream2_skip(&s->gb, tree_size * 2);
node = tree_root; node = tree_root;
bits = *src++; bits = bytestream2_get_byte(&s->gb);
mask = 0x80; mask = 0x80;
for (;;) { for (;;) {
int bit = !!(bits & mask); int bit = !!(bits & mask);
mask >>= 1; mask >>= 1;
node = tree[node*2 + bit]; bytestream2_seek(&tree, start_off + node*2 + bit - eof * 2, SEEK_SET);
node = bytestream2_get_byte(&tree);
if (node == eof) if (node == eof)
break; break;
if (node < eof) { if (node < eof) {
@ -90,49 +91,51 @@ static int xan_unpack_luma(const uint8_t *src, const int src_size,
node = tree_root; node = tree_root;
} }
if (!mask) { if (!mask) {
bits = *src++; if (bytestream2_get_bytes_left(&s->gb) <= 0)
if (src > src_end)
break; break;
bits = bytestream2_get_byteu(&s->gb);
mask = 0x80; mask = 0x80;
} }
} }
return dst != dst_end; return dst != dst_end ? AVERROR_INVALIDDATA : 0;
} }
/* almost the same as in xan_wc3 decoder */ /* almost the same as in xan_wc3 decoder */
static int xan_unpack(uint8_t *dest, const int dest_len, static int xan_unpack(XanContext *s,
const uint8_t *src, const int src_len) uint8_t *dest, const int dest_len)
{ {
uint8_t opcode; uint8_t opcode;
int size; int size;
uint8_t *orig_dest = dest; uint8_t *orig_dest = dest;
const uint8_t *src_end = src + src_len;
const uint8_t *dest_end = dest + dest_len; const uint8_t *dest_end = dest + dest_len;
while (dest < dest_end) { while (dest < dest_end) {
opcode = *src++; if (bytestream2_get_bytes_left(&s->gb) <= 0)
return AVERROR_INVALIDDATA;
opcode = bytestream2_get_byteu(&s->gb);
if (opcode < 0xe0) { if (opcode < 0xe0) {
int size2, back; int size2, back;
if ((opcode & 0x80) == 0) { if ((opcode & 0x80) == 0) {
size = opcode & 3; size = opcode & 3;
back = ((opcode & 0x60) << 3) + *src++ + 1; back = ((opcode & 0x60) << 3) + bytestream2_get_byte(&s->gb) + 1;
size2 = ((opcode & 0x1c) >> 2) + 3; size2 = ((opcode & 0x1c) >> 2) + 3;
} else if ((opcode & 0x40) == 0) { } else if ((opcode & 0x40) == 0) {
size = *src >> 6; size = bytestream2_peek_byte(&s->gb) >> 6;
back = (bytestream_get_be16(&src) & 0x3fff) + 1; back = (bytestream2_get_be16(&s->gb) & 0x3fff) + 1;
size2 = (opcode & 0x3f) + 4; size2 = (opcode & 0x3f) + 4;
} else { } else {
size = opcode & 3; size = opcode & 3;
back = ((opcode & 0x10) << 12) + bytestream_get_be16(&src) + 1; back = ((opcode & 0x10) << 12) + bytestream2_get_be16(&s->gb) + 1;
size2 = ((opcode & 0x0c) << 6) + *src++ + 5; size2 = ((opcode & 0x0c) << 6) + bytestream2_get_byte(&s->gb) + 5;
if (size + size2 > dest_end - dest) if (size + size2 > dest_end - dest)
break; break;
} }
if (src + size > src_end || dest + size + size2 > dest_end || if (dest + size + size2 > dest_end ||
dest - orig_dest + size < back) dest - orig_dest + size < back)
return -1; return -1;
bytestream_get_buffer(&src, dest, size); bytestream2_get_buffer(&s->gb, dest, size);
dest += size; dest += size;
av_memcpy_backptr(dest, back, size2); av_memcpy_backptr(dest, back, size2);
dest += size2; dest += size2;
@ -140,9 +143,9 @@ static int xan_unpack(uint8_t *dest, const int dest_len,
int finish = opcode >= 0xfc; int finish = opcode >= 0xfc;
size = finish ? opcode & 3 : ((opcode & 0x1f) << 2) + 4; size = finish ? opcode & 3 : ((opcode & 0x1f) << 2) + 4;
if (src + size > src_end || dest + size > dest_end) if (dest_end - dest < size)
return -1; return -1;
bytestream_get_buffer(&src, dest, size); bytestream2_get_buffer(&s->gb, dest, size);
dest += size; dest += size;
if (finish) if (finish)
break; break;
@ -151,38 +154,35 @@ static int xan_unpack(uint8_t *dest, const int dest_len,
return dest - orig_dest; return dest - orig_dest;
} }
static int xan_decode_chroma(AVCodecContext *avctx, AVPacket *avpkt) static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off)
{ {
const uint8_t *buf = avpkt->data;
XanContext *s = avctx->priv_data; XanContext *s = avctx->priv_data;
uint8_t *U, *V; uint8_t *U, *V;
unsigned chroma_off;
int val, uval, vval; int val, uval, vval;
int i, j; int i, j;
const uint8_t *src, *src_end; const uint8_t *src, *src_end;
const uint8_t *table; const uint8_t *table;
int mode, offset, dec_size; int mode, offset, dec_size;
chroma_off = AV_RL32(buf + 4);
if (!chroma_off) if (!chroma_off)
return 0; return 0;
if (chroma_off + 10 >= avpkt->size) { if (chroma_off + 4 >= bytestream2_get_bytes_left(&s->gb)) {
av_log(avctx, AV_LOG_ERROR, "Invalid chroma block position\n"); av_log(avctx, AV_LOG_ERROR, "Invalid chroma block position\n");
return -1; return -1;
} }
src = avpkt->data + 4 + chroma_off; bytestream2_seek(&s->gb, chroma_off + 4, SEEK_SET);
table = src + 2; mode = bytestream2_get_le16(&s->gb);
mode = bytestream_get_le16(&src); table = s->gb.buffer;
offset = bytestream_get_le16(&src) * 2; offset = bytestream2_get_le16(&s->gb) * 2;
if (src - avpkt->data >= avpkt->size - offset) { if (offset >= bytestream2_get_bytes_left(&s->gb)) {
av_log(avctx, AV_LOG_ERROR, "Invalid chroma block offset\n"); av_log(avctx, AV_LOG_ERROR, "Invalid chroma block offset\n");
return -1; return -1;
} }
bytestream2_skip(&s->gb, offset);
memset(s->scratch_buffer, 0, s->buffer_size); memset(s->scratch_buffer, 0, s->buffer_size);
dec_size = xan_unpack(s->scratch_buffer, s->buffer_size, src + offset, dec_size = xan_unpack(s, s->scratch_buffer, s->buffer_size);
avpkt->size - offset - (src - avpkt->data));
if (dec_size < 0) { if (dec_size < 0) {
av_log(avctx, AV_LOG_ERROR, "Chroma unpacking failed\n"); av_log(avctx, AV_LOG_ERROR, "Chroma unpacking failed\n");
return -1; return -1;
@ -234,32 +234,27 @@ static int xan_decode_chroma(AVCodecContext *avctx, AVPacket *avpkt)
return 0; return 0;
} }
static int xan_decode_frame_type0(AVCodecContext *avctx, AVPacket *avpkt) static int xan_decode_frame_type0(AVCodecContext *avctx)
{ {
const uint8_t *buf = avpkt->data;
XanContext *s = avctx->priv_data; XanContext *s = avctx->priv_data;
uint8_t *ybuf, *prev_buf, *src = s->scratch_buffer; uint8_t *ybuf, *prev_buf, *src = s->scratch_buffer;
unsigned chroma_off, corr_off; unsigned chroma_off, corr_off;
int cur, last, size; int cur, last;
int i, j; int i, j;
int ret; int ret;
corr_off = AV_RL32(buf + 8); chroma_off = bytestream2_get_le32(&s->gb);
chroma_off = AV_RL32(buf + 4); corr_off = bytestream2_get_le32(&s->gb);
if ((ret = xan_decode_chroma(avctx, avpkt)) != 0) if ((ret = xan_decode_chroma(avctx, chroma_off)) != 0)
return ret; return ret;
size = avpkt->size - 4; if (corr_off >= (s->gb.buffer_end - s->gb.buffer_start)) {
if (corr_off >= avpkt->size) {
av_log(avctx, AV_LOG_WARNING, "Ignoring invalid correction block position\n"); av_log(avctx, AV_LOG_WARNING, "Ignoring invalid correction block position\n");
corr_off = 0; corr_off = 0;
} }
if (corr_off) bytestream2_seek(&s->gb, 12, SEEK_SET);
size = corr_off; ret = xan_unpack_luma(s, src, s->buffer_size >> 1);
if (chroma_off)
size = FFMIN(size, chroma_off);
ret = xan_unpack_luma(buf + 12, size, src, s->buffer_size >> 1);
if (ret) { if (ret) {
av_log(avctx, AV_LOG_ERROR, "Luma decoding failed\n"); av_log(avctx, AV_LOG_ERROR, "Luma decoding failed\n");
return ret; return ret;
@ -295,12 +290,11 @@ static int xan_decode_frame_type0(AVCodecContext *avctx, AVPacket *avpkt)
if (corr_off) { if (corr_off) {
int corr_end, dec_size; int corr_end, dec_size;
corr_end = avpkt->size; corr_end = (s->gb.buffer_end - s->gb.buffer_start);
if (chroma_off > corr_off) if (chroma_off > corr_off)
corr_end = chroma_off; corr_end = chroma_off;
dec_size = xan_unpack(s->scratch_buffer, s->buffer_size, bytestream2_seek(&s->gb, 8 + corr_off, SEEK_SET);
avpkt->data + 8 + corr_off, dec_size = xan_unpack(s, s->scratch_buffer, s->buffer_size);
corr_end - corr_off);
if (dec_size < 0) if (dec_size < 0)
dec_size = 0; dec_size = 0;
for (i = 0; i < dec_size; i++) for (i = 0; i < dec_size; i++)
@ -319,19 +313,19 @@ static int xan_decode_frame_type0(AVCodecContext *avctx, AVPacket *avpkt)
return 0; return 0;
} }
static int xan_decode_frame_type1(AVCodecContext *avctx, AVPacket *avpkt) static int xan_decode_frame_type1(AVCodecContext *avctx)
{ {
const uint8_t *buf = avpkt->data;
XanContext *s = avctx->priv_data; XanContext *s = avctx->priv_data;
uint8_t *ybuf, *src = s->scratch_buffer; uint8_t *ybuf, *src = s->scratch_buffer;
int cur, last; int cur, last;
int i, j; int i, j;
int ret; int ret;
if ((ret = xan_decode_chroma(avctx, avpkt)) != 0) if ((ret = xan_decode_chroma(avctx, bytestream2_get_le32(&s->gb))) != 0)
return ret; return ret;
ret = xan_unpack_luma(buf + 16, avpkt->size - 16, src, bytestream2_seek(&s->gb, 16, SEEK_SET);
ret = xan_unpack_luma(s, src,
s->buffer_size >> 1); s->buffer_size >> 1);
if (ret) { if (ret) {
av_log(avctx, AV_LOG_ERROR, "Luma decoding failed\n"); av_log(avctx, AV_LOG_ERROR, "Luma decoding failed\n");
@ -381,13 +375,14 @@ static int xan_decode_frame(AVCodecContext *avctx,
return ret; return ret;
} }
ftype = AV_RL32(avpkt->data); bytestream2_init(&s->gb, avpkt->data, avpkt->size);
ftype = bytestream2_get_le32(&s->gb);
switch (ftype) { switch (ftype) {
case 0: case 0:
ret = xan_decode_frame_type0(avctx, avpkt); ret = xan_decode_frame_type0(avctx);
break; break;
case 1: case 1:
ret = xan_decode_frame_type1(avctx, avpkt); ret = xan_decode_frame_type1(avctx);
break; break;
default: default:
av_log(avctx, AV_LOG_ERROR, "Unknown frame type %d\n", ftype); av_log(avctx, AV_LOG_ERROR, "Unknown frame type %d\n", ftype);