avcodec/mpeg12enc: Don't write invalid MPEG-1 slice headers

The valid values for slice_start_code are 0x1..0xAF, which implies
that one can't start a slice with row nb > 174 (zero-based).
This problem can be encountered with files of sufficient height
(more than 2800 pixels) either by using the slice option or by
imposing an RTP payload limit.

Fix this by making the last slice start on the maximum allowed
slice row if necessary and divide the first 174 rows to the remaining
slices. This will impede parallelism both in the decoder and encoder,
but that is unavoidable.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

libavcodec/mpeg12enc.c

@@ -25,6 +25,7 @@
  * MPEG-1/2 encoder
  */
 
+#include <assert.h>
 #include <stdint.h>
 
 #include "config.h"
@@ -324,6 +325,7 @@ void ff_mpeg1_encode_slice_header(MpegEncContext *s)
         /* slice_vertical_position_extension */
         put_bits(&s->pb, 3, s->mb_y >> 7);
     } else {
+        av_assert1(s->mb_y <= SLICE_MAX_START_CODE - SLICE_MIN_START_CODE);
         put_header(s, SLICE_MIN_START_CODE + s->mb_y);
     }
     put_qscale(s);
@@ -1128,6 +1130,30 @@ static av_cold int encode_init(AVCodecContext *avctx)
     if (ret < 0)
         return ret;
 
+    if (avctx->codec_id == AV_CODEC_ID_MPEG1VIDEO &&
+        s->thread_context[s->slice_context_count - 1]->start_mb_y >
+            SLICE_MAX_START_CODE - SLICE_MIN_START_CODE) {
+        // MPEG-1 slices must not start at a MB row number that would make
+        // their start code > SLICE_MAX_START_CODE. So make the last slice
+        // bigger if needed and evenly distribute the first 174 rows.
+        static_assert(MAX_THREADS <= 1 + SLICE_MAX_START_CODE - SLICE_MIN_START_CODE,
+                      "With more than 175 slice contexts, we have to handle "
+                      "the case in which there is no work to do for some "
+                      "slice contexts.");
+        const int mb_height = SLICE_MAX_START_CODE - SLICE_MIN_START_CODE;
+        const int nb_slices = s->slice_context_count - 1;
+
+        s->thread_context[nb_slices]->start_mb_y = mb_height;
+
+        av_assert1(nb_slices >= 1);
+        for (int i = 0; i < nb_slices; i++) {
+            s->thread_context[i]->start_mb_y =
+                (mb_height * (i    ) + nb_slices / 2) / nb_slices;
+            s->thread_context[i]->end_mb_y   =
+                (mb_height * (i + 1) + nb_slices / 2) / nb_slices;
+        }
+    }
+
     if (find_frame_rate_index(avctx, mpeg12) < 0) {
         if (avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
             av_log(avctx, AV_LOG_ERROR, "MPEG-1/2 does not support %d/%d fps\n",

libavcodec/mpegvideo_enc.c

@@ -3077,7 +3077,9 @@ static int encode_thread(AVCodecContext *c, void *arg){
                 case AV_CODEC_ID_MPEG2VIDEO:
                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
                 case AV_CODEC_ID_MPEG1VIDEO:
-                    if(s->mb_skip_run) is_gob_start=0;
+                    if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO && s->mb_y >= 175 ||
+                        s->mb_skip_run)
+                        is_gob_start=0;
                     break;
                 case AV_CODEC_ID_MJPEG:
                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;