avformat/mxfenc: Ensure frame offset in valid range

Fix assert failure.
Fix #11666.

Signed-off-by: Zhao Zhili <zhilizhao@tencent.com>

libavformat/mxfenc.c

@@ -1983,7 +1983,19 @@ static unsigned klv_fill_size(uint64_t size)
         return pad & (KAG_SIZE-1);
 }
 
-static void mxf_write_index_table_segment(AVFormatContext *s)
+static int mxf_check_frame_offset(AVFormatContext *s, int offset)
+{
+    if (offset >= INT8_MIN && offset <= INT8_MAX)
+        return 0;
+
+    av_log(s, AV_LOG_ERROR, "Current implementation requires frame offset (%d) "
+            "to be within the range of [%d,%d]. Please reduce encoder GOP size, "
+            "or add support for wider frame offset ranges.\n",
+            offset, INT8_MIN, INT8_MAX);
+    return AVERROR_PATCHWELCOME;
+}
+
+static int mxf_write_index_table_segment(AVFormatContext *s)
 {
     MXFContext *mxf = s->priv_data;
     AVIOContext *pb = s->pb;
@@ -1992,11 +2004,12 @@ static void mxf_write_index_table_segment(AVFormatContext *s)
     int prev_non_b_picture = 0;
     int audio_frame_size = 0;
     int64_t pos;
+    int err;
 
     av_log(s, AV_LOG_DEBUG, "edit units count %d\n", mxf->edit_units_count);
 
     if (!mxf->edit_units_count && !mxf->edit_unit_byte_count)
-        return;
+        return 0;
 
     avio_write(pb, index_table_segment_key, 16);
 
@@ -2095,15 +2108,26 @@ static void mxf_write_index_table_segment(AVFormatContext *s)
                     if (j == mxf->edit_units_count)
                         av_log(s, AV_LOG_WARNING, "missing frames\n");
                     temporal_offset = j - key_index - pic_num_in_gop;
+                    err = mxf_check_frame_offset(s, temporal_offset);
+                    if (err < 0)
+                        return err;
                 }
             }
             avio_w8(pb, temporal_offset);
 
             if ((mxf->index_entries[i].flags & 0x30) == 0x30) { // back and forward prediction
+                int offset = mxf->last_key_index - i;
+                err = mxf_check_frame_offset(s, offset);
+                if (err < 0)
+                    return err;
                 sc->b_picture_count = FFMAX(sc->b_picture_count, i - prev_non_b_picture);
-                avio_w8(pb, mxf->last_key_index - i);
+                avio_w8(pb, offset);
             } else {
-                avio_w8(pb, key_index - i); // key frame offset
+                int offset = key_index - i;
+                err = mxf_check_frame_offset(s, offset);
+                if (err < 0)
+                    return err;
+                avio_w8(pb, offset); // key frame offset
                 if ((mxf->index_entries[i].flags & 0x20) == 0x20) // only forward
                     mxf->last_key_index = key_index;
                 prev_non_b_picture = i;
@@ -2127,6 +2151,8 @@ static void mxf_write_index_table_segment(AVFormatContext *s)
     }
 
     mxf_update_klv_size(pb, pos);
+
+    return 0;
 }
 
 static void mxf_write_klv_fill(AVFormatContext *s)
@@ -3354,7 +3380,9 @@ static int mxf_write_packet(AVFormatContext *s, AVPacket *pkt)
             if ((err = mxf_write_partition(s, 1, 2, header_open_partition_key, 1)) < 0)
                 return err;
             mxf_write_klv_fill(s);
-            mxf_write_index_table_segment(s);
+            err = mxf_write_index_table_segment(s);
+            if (err < 0)
+                return err;
         } else {
             if ((err = mxf_write_partition(s, 0, 0, header_open_partition_key, 1)) < 0)
                 return err;
@@ -3370,7 +3398,9 @@ static int mxf_write_packet(AVFormatContext *s, AVPacket *pkt)
             if ((err = mxf_write_partition(s, 1, 2, body_partition_key, 0)) < 0)
                 return err;
             mxf_write_klv_fill(s);
-            mxf_write_index_table_segment(s);
+            err = mxf_write_index_table_segment(s);
+            if (err < 0)
+                return err;
         }
 
         mxf_write_klv_fill(s);
@@ -3455,7 +3485,9 @@ static int mxf_write_footer(AVFormatContext *s)
         if ((err = mxf_write_partition(s, 0, 2, footer_partition_key, 0)) < 0)
             return err;
         mxf_write_klv_fill(s);
-        mxf_write_index_table_segment(s);
+        err = mxf_write_index_table_segment(s);
+        if (err < 0)
+            return err;
     }
 
     mxf_write_klv_fill(s);
@@ -3474,7 +3506,9 @@ static int mxf_write_footer(AVFormatContext *s)
             if ((err = mxf_write_partition(s, 1, 2, header_closed_partition_key, 1)) < 0)
                 return err;
             mxf_write_klv_fill(s);
-            mxf_write_index_table_segment(s);
+            err = mxf_write_index_table_segment(s);
+            if (err < 0)
+                return err;
         } else {
             if ((err = mxf_write_partition(s, 0, 0, header_closed_partition_key, 1)) < 0)
                 return err;