From 0ccddbad200c1d9439c5a836501917d515cddf76 Mon Sep 17 00:00:00 2001 From: Andreas Cadhalpun Date: Sat, 19 Nov 2016 14:21:11 +0100 Subject: [PATCH] smacker: limit recursion depth of smacker_decode_bigtree This fixes segmentation faults due to stack-overflow caused by too deep recursion. Reviewed-by: Michael Niedermayer Signed-off-by: Andreas Cadhalpun Signed-off-by: Sean McGovern --- libavcodec/smacker.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c index 7deccffa54..636e3b48e3 100644 --- a/libavcodec/smacker.c +++ b/libavcodec/smacker.c @@ -44,6 +44,7 @@ #define SMKTREE_BITS 9 #define SMK_NODE 0x80000000 #define SMKTREE_DECODE_MAX_RECURSION 32 +#define SMKTREE_DECODE_BIG_MAX_RECURSION 500 typedef struct SmackVContext { AVCodecContext *avctx; @@ -133,8 +134,14 @@ static int smacker_decode_tree(BitstreamContext *bc, HuffContext *hc, * Decode header tree */ static int smacker_decode_bigtree(BitstreamContext *bc, HuffContext *hc, - DBCtx *ctx) + DBCtx *ctx, int length) { + // Larger length can cause segmentation faults due to too deep recursion. + if (length > SMKTREE_DECODE_BIG_MAX_RECURSION) { + av_log(NULL, AV_LOG_ERROR, "Maximum bigtree recursion level exceeded.\n"); + return AVERROR_INVALIDDATA; + } + if (hc->current + 1 >= hc->length) { av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n"); return AVERROR_INVALIDDATA; @@ -163,12 +170,12 @@ static int smacker_decode_bigtree(BitstreamContext *bc, HuffContext *hc, int r = 0, r_new, t; t = hc->current++; - r = smacker_decode_bigtree(bc, hc, ctx); + r = smacker_decode_bigtree(bc, hc, ctx, length + 1); if(r < 0) return r; hc->values[t] = SMK_NODE | r; r++; - r_new = smacker_decode_bigtree(bc, hc, ctx); + r_new = smacker_decode_bigtree(bc, hc, ctx, length + 1); if (r_new < 0) return r_new; return r + r_new; @@ -269,7 +276,7 @@ static int smacker_decode_header_tree(SmackVContext *smk, BitstreamContext *bc, goto error; } - if ((res = smacker_decode_bigtree(bc, &huff, &ctx)) < 0) + if ((res = smacker_decode_bigtree(bc, &huff, &ctx, 0)) < 0) err = res; bitstream_skip(bc, 1); if(ctx.last[0] == -1) ctx.last[0] = huff.current++;