avcodec/ffv1: Fix remap and float with golomb rice

Sponsored-by: Sovereign Tech Fund
Reviewed-by: Lynne <dev@lynne.ee>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/ffv1dec.c

@@ -243,6 +243,37 @@ static void slice_set_damaged(FFV1Context *f, FFV1SliceContext *sc)
         f->frame_damaged = 1;
 }
 
+static int decode_remap(FFV1Context *f, FFV1SliceContext *sc)
+{
+    int transparency = f->transparency;
+
+    for (int p= 0; p<3 + transparency; p++) {
+        int j = 0;
+        int lu = 0;
+        uint8_t state[2][32];
+        memset(state, 128, sizeof(state));
+
+        for (int i= 0; i<65536; i++) {
+            int run = get_symbol_inline(&sc->c, state[lu], 0);
+            if (run > 65536U - i)
+                return AVERROR_INVALIDDATA;
+            if (lu) {
+                lu ^= !run;
+                while (run--) {
+                    sc->fltmap[p][j++] = i ^ ((i&0x8000) ? 0 : 0x7FFF);
+                    i++;
+                }
+            } else {
+                i += run;
+                if (i != 65536)
+                    sc->fltmap[p][j++] = i ^ ((i&0x8000) ? 0 : 0x7FFF);
+                lu ^= !run;
+            }
+        }
+    }
+    return 0;
+}
+
 static int decode_slice(AVCodecContext *c, void *arg)
 {
     FFV1Context *f    = c->priv_data;
@@ -285,6 +316,12 @@ static int decode_slice(AVCodecContext *c, void *arg)
     x      = sc->slice_x;
     y      = sc->slice_y;
 
+    if (sc->remap) {
+        ret = decode_remap(f, sc);
+        if (ret < 0)
+            return ret;
+    }
+
     if (ac == AC_GOLOMB_RICE) {
         if (f->combined_version >= 0x30002)
             get_rac(&sc->c, (uint8_t[]) { 129 });

libavcodec/ffv1dec_template.c

@@ -155,33 +155,6 @@ static int RENAME(decode_rgb_frame)(FFV1Context *f, FFV1SliceContext *sc,
 
     memset(RENAME(sc->sample_buffer), 0, 8 * (w + 6) * sizeof(*RENAME(sc->sample_buffer)));
 
-    if (sc->remap) {
-        for (int p= 0; p<3 + transparency; p++) {
-            int j = 0;
-            int lu = 0;
-            uint8_t state[2][32];
-            memset(state, 128, sizeof(state));
-
-            for (int i= 0; i<65536; i++) {
-                int run = get_symbol_inline(&sc->c, state[lu], 0);
-                if (run > 65536U - i)
-                    return AVERROR_INVALIDDATA;
-                if (lu) {
-                    lu ^= !run;
-                    while (run--) {
-                        sc->fltmap[p][j++] = i ^ ((i&0x8000) ? 0 : 0x7FFF);
-                        i++;
-                    }
-                } else {
-                    i += run;
-                    if (i != 65536)
-                        sc->fltmap[p][j++] = i ^ ((i&0x8000) ? 0 : 0x7FFF);
-                    lu ^= !run;
-                }
-            }
-        }
-    }
-
     for (y = 0; y < h; y++) {
         for (p = 0; p < 3 + transparency; p++) {
             int ret;

libavcodec/ffv1enc.c

@@ -1101,6 +1101,36 @@ static void choose_rct_params(const FFV1Context *f, FFV1SliceContext *sc,
     sc->slice_rct_ry_coef = rct_y_coeff[best][0];
 }
 
+static void encode_remap(FFV1Context *f, FFV1SliceContext *sc)
+{
+    int transparency = f->transparency;
+
+    for (int p= 0; p<3 + transparency; p++) {
+        int j = 0;
+        int lu = 0;
+        uint8_t state[2][32];
+        int run = 0;
+        memset(state, 128, sizeof(state));
+        for (int i= 0; i<65536; i++) {
+            int ri = i ^ ((i&0x8000) ? 0 : 0x7FFF);
+            int u = sc->fltmap[p][ri];
+            sc->fltmap[p][ri] = j;
+            j+= u;
+
+            if (lu == u) {
+                run ++;
+            } else {
+                put_symbol_inline(&sc->c, state[lu], run, 0, NULL, NULL);
+                if (run == 0)
+                    lu = u;
+                run = 0;
+            }
+        }
+        if (run)
+            put_symbol(&sc->c, state[lu], run, 0);
+    }
+}
+
 static int encode_slice(AVCodecContext *c, void *arg)
 {
     FFV1SliceContext *sc = arg;
@@ -1133,6 +1163,18 @@ retry:
     if (f->version > 2) {
         encode_slice_header(f, sc);
     }
+
+    if (sc->remap) {
+        if (f->colorspace == 0) {
+            av_assert0(0);
+        } else if (f->use32bit) {
+            load_rgb_frame32(f, sc, planes, width, height, p->linesize);
+        } else
+            load_rgb_frame  (f, sc, planes, width, height, p->linesize);
+
+        encode_remap(f, sc);
+    }
+
     if (ac == AC_GOLOMB_RICE) {
         sc->ac_byte_count = f->version > 2 || (!x && !y) ? ff_rac_terminate(&sc->c, f->version > 2) : 0;
         init_put_bits(&sc->pb,

libavcodec/ffv1enc_template.c

@@ -127,6 +127,40 @@ RENAME(encode_line)(FFV1Context *f, FFV1SliceContext *sc,
     return 0;
 }
 
+static void RENAME(load_rgb_frame)(FFV1Context *f, FFV1SliceContext *sc,
+                                  const uint8_t *src[4],
+                                  int w, int h, const int stride[4])
+{
+    int x, y;
+    int transparency = f->transparency;
+
+    memset(sc->fltmap, 0, sizeof(sc->fltmap));
+
+    for (y = 0; y < h; y++) {
+        for (x = 0; x < w; x++) {
+            int b, g, r, av_uninit(a);
+
+            if (sizeof(TYPE) == 4 || transparency) {
+                g = *((const uint16_t *)(src[0] + x*2 + stride[0]*y));
+                b = *((const uint16_t *)(src[1] + x*2 + stride[1]*y));
+                r = *((const uint16_t *)(src[2] + x*2 + stride[2]*y));
+                if (transparency)
+                    a = *((const uint16_t *)(src[3] + x*2 + stride[3]*y));
+            } else {
+                b = *((const uint16_t *)(src[0] + x*2 + stride[0]*y));
+                g = *((const uint16_t *)(src[1] + x*2 + stride[1]*y));
+                r = *((const uint16_t *)(src[2] + x*2 + stride[2]*y));
+            }
+
+            sc->fltmap[0][r] = 1;
+            sc->fltmap[1][g] = 1;
+            sc->fltmap[2][b] = 1;
+            if (transparency)
+                sc->fltmap[3][a] = 1;
+        }
+    }
+}
+
 static int RENAME(encode_rgb_frame)(FFV1Context *f, FFV1SliceContext *sc,
                                     const uint8_t *src[4],
                                     int w, int h, const int stride[4], int ac)
@@ -147,58 +181,6 @@ static int RENAME(encode_rgb_frame)(FFV1Context *f, FFV1SliceContext *sc,
     memset(RENAME(sc->sample_buffer), 0, ring_size * MAX_PLANES *
            (w + 6) * sizeof(*RENAME(sc->sample_buffer)));
 
-    if (sc->remap) {
-        memset(sc->fltmap, 0, sizeof(sc->fltmap));
-
-        for (y = 0; y < h; y++) {
-            for (x = 0; x < w; x++) {
-                int b, g, r, av_uninit(a);
-
-                if (sizeof(TYPE) == 4 || transparency) {
-                    g = *((const uint16_t *)(src[0] + x*2 + stride[0]*y));
-                    b = *((const uint16_t *)(src[1] + x*2 + stride[1]*y));
-                    r = *((const uint16_t *)(src[2] + x*2 + stride[2]*y));
-                    if (transparency)
-                        a = *((const uint16_t *)(src[3] + x*2 + stride[3]*y));
-                } else {
-                    b = *((const uint16_t *)(src[0] + x*2 + stride[0]*y));
-                    g = *((const uint16_t *)(src[1] + x*2 + stride[1]*y));
-                    r = *((const uint16_t *)(src[2] + x*2 + stride[2]*y));
-                }
-
-                sc->fltmap[0][r] = 1;
-                sc->fltmap[1][g] = 1;
-                sc->fltmap[2][b] = 1;
-                if (transparency)
-                    sc->fltmap[3][a] = 1;
-            }
-        }
-        for (int p= 0; p<3 + transparency; p++) {
-            int j = 0;
-            int lu = 0;
-            uint8_t state[2][32];
-            int run = 0;
-            memset(state, 128, sizeof(state));
-            for (int i= 0; i<65536; i++) {
-                int ri = i ^ ((i&0x8000) ? 0 : 0x7FFF);
-                int u = sc->fltmap[p][ri];
-                sc->fltmap[p][ri] = j;
-                j+= u;
-
-                if (lu == u) {
-                    run ++;
-                } else {
-                    put_symbol_inline(&sc->c, state[lu], run, 0, NULL, NULL);
-                    if (run == 0)
-                        lu = u;
-                    run = 0;
-                }
-            }
-            if (run)
-                put_symbol(&sc->c, state[lu], run, 0);
-        }
-    }
-
     for (y = 0; y < h; y++) {
         for (i = 0; i < ring_size; i++)
             for (p = 0; p < MAX_PLANES; p++)