From eba1ff31304e407db3cefd7532108408f364367b Mon Sep 17 00:00:00 2001 From: Xi Wang Date: Fri, 15 Mar 2013 06:31:21 -0400 Subject: [PATCH 1/3] atrac3: avoid oversized shifting in decode_bytes() When `off' is 0, `0x537F6103 << 32' in the following expression invokes undefined behavior, the result of which is not necessarily 0. (0x537F6103 >> (off * 8)) | (0x537F6103 << (32 - (off * 8))) Avoid oversized shifting. CC: libav-stable@libav.org Signed-off-by: Xi Wang Signed-off-by: Luca Barbato --- libavcodec/atrac3.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libavcodec/atrac3.c b/libavcodec/atrac3.c index da0bff0bf3..a52c5c618a 100644 --- a/libavcodec/atrac3.c +++ b/libavcodec/atrac3.c @@ -163,7 +163,10 @@ static int decode_bytes(const uint8_t *input, uint8_t *out, int bytes) off = (intptr_t)input & 3; buf = (const uint32_t *)(input - off); - c = av_be2ne32((0x537F6103 >> (off * 8)) | (0x537F6103 << (32 - (off * 8)))); + if (off) + c = av_be2ne32((0x537F6103U >> (off * 8)) | (0x537F6103U << (32 - (off * 8)))); + else + c = av_be2ne32(0x537F6103U); bytes += 3 + off; for (i = 0; i < bytes / 4; i++) output[i] = c ^ buf[i]; From 8425d693eefbedbb41f91735614d41067695aa37 Mon Sep 17 00:00:00 2001 From: Xi Wang Date: Fri, 15 Mar 2013 07:11:47 -0400 Subject: [PATCH 2/3] flacdec: simplify bounds checking in flac_probe() Simplify `p->buf > p->buf + p->buf_size - 4' as `p->buf_size < 4'. Avoid a possible out-of-bounds pointer, which is undefined behavior in C. CC: libav-stable@libav.org Signed-off-by: Xi Wang Signed-off-by: Luca Barbato --- libavformat/flacdec.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/libavformat/flacdec.c b/libavformat/flacdec.c index 7aaec9eb26..1edd88037a 100644 --- a/libavformat/flacdec.c +++ b/libavformat/flacdec.c @@ -279,11 +279,9 @@ static int flac_read_header(AVFormatContext *s) static int flac_probe(AVProbeData *p) { - uint8_t *bufptr = p->buf; - uint8_t *end = p->buf + p->buf_size; - - if(bufptr > end-4 || memcmp(bufptr, "fLaC", 4)) return 0; - else return AVPROBE_SCORE_MAX/2; + if (p->buf_size < 4 || memcmp(p->buf, "fLaC", 4)) + return 0; + return AVPROBE_SCORE_MAX/2; } AVInputFormat ff_flac_demuxer = { From ca6c3f2c53be70aa3c38e8f1292809db89ea1ba6 Mon Sep 17 00:00:00 2001 From: Xi Wang Date: Fri, 15 Mar 2013 06:59:22 -0400 Subject: [PATCH 3/3] lzo: fix overflow checking in copy_backptr() The check `src > dst' in the form `&c->out[-back] > c->out' invokes pointer overflow, which is undefined behavior in C. Remove the check. Also replace `&c->out[-back] < c->out_start' with a safe form `c->out - c->out_start < back' to avoid overflow. CC: libav-stable@libav.org Signed-off-by: Xi Wang Signed-off-by: Luca Barbato --- libavutil/lzo.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libavutil/lzo.c b/libavutil/lzo.c index eff3cd2333..5c5ebc850a 100644 --- a/libavutil/lzo.c +++ b/libavutil/lzo.c @@ -110,9 +110,8 @@ static inline void copy(LZOContext *c, int cnt) */ static inline void copy_backptr(LZOContext *c, int back, int cnt) { - register const uint8_t *src = &c->out[-back]; register uint8_t *dst = c->out; - if (src < c->out_start || src > dst) { + if (dst - c->out_start < back) { c->error |= AV_LZO_INVALID_BACKPTR; return; }