avcodec/vlc, bitstream: Fix multi VLC with uint8_t syms on BE

VLC_MULTI_ELEM contains an uint8_t array that is supposed
to be treated as an array of uint16_t when the used symbols
have a size of two; otherwise it should be treated as just
an array of uint8_t, but it was not always treated that way:

vlc_multi_gen() initialized the first entry of the array
by writing the symbol via AV_WN16; on big endian systems,
the intended value was instead written into the second entry
of the array (where it would likely be overwritten lateron
during initialization).

read_vlc_multi() also treated this case incorrectly: In case
the code is so long that it needs a classical multi-stage lookup,
the symbol has been written to the destination as if via AV_WN16.
On little endian systems, this sets the correct first symbol and
clobbers (zeroes) the next one, but the next one will be overwritten
lateron anyway, so it won't be recognized. But on big-endian systems,
the first symbol will be set to zero and the actually read symbol
will be put into the slot for the next one (where it will be overwritten
lateron).

This commit fixes this; this fixes the magicyuv and utvideo FATE-tests
on big endian arches.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
(cherry picked from commit 4ab82d2fb6)

libavcodec/bitstream_template.h

@@ -536,7 +536,8 @@ static inline int BS_FUNC(read_vlc)(BSCTX *bc, const VLCElem *table,
 static inline int BS_FUNC(read_vlc_multi)(BSCTX *bc, uint8_t dst[8],
                                           const VLC_MULTI_ELEM *const Jtable,
                                           const VLCElem *const table,
-                                          const int bits, const int max_depth)
+                                          const int bits, const int max_depth,
+                                          const int symbols_size)
 {
     unsigned idx = BS_FUNC(peek)(bc, bits);
     int ret, nb_bits, code, n = Jtable[idx].len;
@@ -554,6 +555,9 @@ static inline int BS_FUNC(read_vlc_multi)(BSCTX *bc, uint8_t dst[8],
                 code = BS_FUNC(priv_set_idx)(bc, code, &n, &nb_bits, table);
             }
         }
+        if (symbols_size == 1)
+            *dst = code;
+        else
             AV_WN16(dst, code);
         ret = n > 0;
     }

libavcodec/get_bits.h

@@ -667,7 +667,8 @@ static av_always_inline int get_vlc2(GetBitContext *s, const VLCElem *table,
 static inline int get_vlc_multi(GetBitContext *s, uint8_t *dst,
                                 const VLC_MULTI_ELEM *const Jtable,
                                 const VLCElem *const table,
-                                const int bits, const int max_depth)
+                                const int bits, const int max_depth,
+                                const int symbols_size)
 {
     dst[0] = get_vlc2(s, table, bits, max_depth);
     return 1;

libavcodec/magicyuv.c

@@ -124,7 +124,7 @@ static void magicyuv_median_pred16(uint16_t *dst, const uint16_t *src1,
     x = 0; \
     for (; CACHED_BITSTREAM_READER && x < width-c && get_bits_left(&gb) > 0;) {\
         ret = get_vlc_multi(&gb, (uint8_t *)dst + x * b, multi, \
-                            vlc, vlc_bits, 3); \
+                            vlc, vlc_bits, 3, b); \
         if (ret <= 0) \
             return AVERROR_INVALIDDATA; \
         x += ret; \

libavcodec/utvideodec.c

@@ -120,7 +120,7 @@ static int build_huff(UtvideoContext *c, const uint8_t *src, VLC *vlc,
     i = 0; \
     for (; CACHED_BITSTREAM_READER && i < width-end && get_bits_left(&gb) > 0;) {\
         ret = get_vlc_multi(&gb, (uint8_t *)buf + i * b, multi.table, \
-                            vlc.table, VLC_BITS, 3); \
+                            vlc.table, VLC_BITS, 3, b); \
         if (ret > 0) \
             i += ret; \
         if (ret <= 0) \

libavcodec/vlc.c

@@ -499,7 +499,10 @@ static int vlc_multi_gen(VLC_MULTI_ELEM *table, const VLC *single,
     for (int j = 0; j < 1<<numbits; j++) {
         table[j].len = single->table[j].len;
         table[j].num = single->table[j].len > 0 ? 1 : 0;
+        if (is16bit)
             AV_WN16(table[j].val, single->table[j].sym);
+        else
+            table[j].val[0] = single->table[j].sym;
     }
 
     add_level(table, is16bit, nb_codes, numbits, buf,