avcodec/iff: add support for vertical word compression in ILBM

Signed-off-by: Paul B Mahol <onemda@gmail.com>

libavcodec/iff.c

@@ -413,7 +413,7 @@ static av_cold int decode_init(AVCodecContext *avctx)
     if ((err = av_image_check_size(avctx->width, avctx->height, 0, avctx)))
         return err;
     s->planesize = FFALIGN(avctx->width, 16) >> 3; // Align plane size in bits to word-boundary
-    s->planebuf  = av_malloc(s->planesize + AV_INPUT_BUFFER_PADDING_SIZE);
+    s->planebuf  = av_malloc(s->planesize * avctx->height + AV_INPUT_BUFFER_PADDING_SIZE);
     if (!s->planebuf)
         return AVERROR(ENOMEM);
 
@@ -557,6 +557,76 @@ static int decode_byterun(uint8_t *dst, int dst_size,
     return bytestream2_tell(gb);
 }
 
+static int decode_byterun2(uint8_t *dst, int height, int line_size,
+                           GetByteContext *gb)
+{
+    GetByteContext cmds;
+    unsigned count;
+    int i, y_pos = 0, x_pos = 0;
+
+    if (bytestream2_get_be32(gb) != MKBETAG('V', 'D', 'A', 'T'))
+        return 0;
+
+    bytestream2_skip(gb, 4);
+    count = bytestream2_get_be16(gb) - 2;
+    if (bytestream2_get_bytes_left(gb) < count)
+        return 0;
+
+    bytestream2_init(&cmds, gb->buffer, count);
+    bytestream2_skip(gb, count);
+
+    for (i = 0; i < count && x_pos < line_size; i++) {
+        int8_t cmd = bytestream2_get_byte(&cmds);
+        int l, r;
+
+        if (cmd == 0) {
+            l = bytestream2_get_be16(gb);
+            while (l-- > 0 && x_pos < line_size) {
+                dst[x_pos + y_pos   * line_size    ] = bytestream2_get_byte(gb);
+                dst[x_pos + y_pos++ * line_size + 1] = bytestream2_get_byte(gb);
+                if (y_pos >= height) {
+                    y_pos  = 0;
+                    x_pos += 2;
+                }
+            }
+        } else if (cmd < 0) {
+            l = -cmd;
+            while (l-- > 0 && x_pos < line_size) {
+                dst[x_pos + y_pos   * line_size    ] = bytestream2_get_byte(gb);
+                dst[x_pos + y_pos++ * line_size + 1] = bytestream2_get_byte(gb);
+                if (y_pos >= height) {
+                    y_pos  = 0;
+                    x_pos += 2;
+                }
+            }
+        } else if (cmd == 1) {
+            l = bytestream2_get_be16(gb);
+            r = bytestream2_get_be16(gb);
+            while (l-- > 0 && x_pos < line_size) {
+                dst[x_pos + y_pos   * line_size    ] = r >> 8;
+                dst[x_pos + y_pos++ * line_size + 1] = r & 0xFF;
+                if (y_pos >= height) {
+                    y_pos  = 0;
+                    x_pos += 2;
+                }
+            }
+        } else {
+            l = cmd;
+            r = bytestream2_get_be16(gb);
+            while (l-- > 0 && x_pos < line_size) {
+                dst[x_pos + y_pos   * line_size    ] = r >> 8;
+                dst[x_pos + y_pos++ * line_size + 1] = r & 0xFF;
+                if (y_pos >= height) {
+                    y_pos  = 0;
+                    x_pos += 2;
+                }
+            }
+        }
+    }
+
+    return bytestream2_tell(gb);
+}
+
 #define DECODE_RGBX_COMMON(type) \
     if (!length) { \
         length = bytestream2_get_byte(gb); \
@@ -1623,6 +1693,19 @@ static int decode_frame(AVCodecContext *avctx,
                 return unsupported(avctx);
         }
         break;
+    case 0x2:
+        if (avctx->codec_tag == MKTAG('I', 'L', 'B', 'M') && avctx->pix_fmt == AV_PIX_FMT_PAL8) {
+            for (plane = 0; plane < s->bpp; plane++) {
+                decode_byterun2(s->planebuf, avctx->height, s->planesize, gb);
+                for (y = 0; y < avctx->height; y++) {
+                    uint8_t *row = &frame->data[0][y * frame->linesize[0]];
+                    decodeplane8(row, s->planebuf + s->planesize * y, s->planesize, plane);
+                }
+            }
+        } else {
+            return unsupported(avctx);
+        }
+        break;
     case 0x4:
         if (avctx->codec_tag == MKTAG('R', 'G', 'B', '8') && avctx->pix_fmt == AV_PIX_FMT_RGB32)
             decode_rgb8(gb, frame->data[0], avctx->width, avctx->height, frame->linesize[0]);