qsvenc: write a53 caption data to SEI

Signed-off-by: Will Kelleher <wkelleher@gogoair.com>
Previous version reviewed-by: Ivan Uskov <ivan.uskov@nablet.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/qsv_internal.h

@@ -45,6 +45,8 @@
 
 #define ASYNC_DEPTH_DEFAULT 4       // internal parallelism
 
+#define QSV_MAX_ENC_PAYLOAD 2       // # of mfxEncodeCtrl payloads supported
+
 #define QSV_VERSION_ATLEAST(MAJOR, MINOR)   \
     (MFX_VERSION_MAJOR > (MAJOR) ||         \
      MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR))
@@ -52,6 +54,7 @@
 typedef struct QSVFrame {
     AVFrame *frame;
     mfxFrameSurface1 *surface;
+    mfxEncodeCtrl enc_ctrl;
 
     mfxFrameSurface1 surface_internal;
 

libavcodec/qsvenc.c

@@ -30,6 +30,7 @@
 #include "libavutil/log.h"
 #include "libavutil/time.h"
 #include "libavutil/imgutils.h"
+#include "libavcodec/bytestream.h"
 
 #include "avcodec.h"
 #include "internal.h"
@@ -753,12 +754,26 @@ int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q)
     return 0;
 }
 
+static void free_encoder_ctrl_payloads(mfxEncodeCtrl* enc_ctrl)
+{
+    if (enc_ctrl) {
+        int i;
+        for (i = 0; i < enc_ctrl->NumPayload && i < QSV_MAX_ENC_PAYLOAD; i++) {
+            mfxPayload* pay = enc_ctrl->Payload[i];
+            av_free(enc_ctrl->Payload[i]->Data);
+            av_free(pay);
+        }
+        enc_ctrl->NumPayload = 0;
+    }
+}
+
 static void clear_unused_frames(QSVEncContext *q)
 {
     QSVFrame *cur = q->work_frames;
     while (cur) {
         if (cur->surface && !cur->surface->Data.Locked) {
             cur->surface = NULL;
+            free_encoder_ctrl_payloads(&cur->enc_ctrl);
             av_frame_unref(cur->frame);
         }
         cur = cur->next;
@@ -791,6 +806,11 @@ static int get_free_frame(QSVEncContext *q, QSVFrame **f)
         av_freep(&frame);
         return AVERROR(ENOMEM);
     }
+    frame->enc_ctrl.Payload = av_mallocz(sizeof(mfxPayload*) * QSV_MAX_ENC_PAYLOAD);
+    if (!frame->enc_ctrl.Payload) {
+        av_freep(&frame);
+        return AVERROR(ENOMEM);
+    }
     *last = frame;
 
     *f = frame;
@@ -799,7 +819,7 @@ static int get_free_frame(QSVEncContext *q, QSVFrame **f)
 }
 
 static int submit_frame(QSVEncContext *q, const AVFrame *frame,
-                        mfxFrameSurface1 **surface)
+                        QSVFrame **new_frame)
 {
     QSVFrame *qf;
     int ret;
@@ -860,7 +880,7 @@ static int submit_frame(QSVEncContext *q, const AVFrame *frame,
 
     qf->surface->Data.TimeStamp = av_rescale_q(frame->pts, q->avctx->time_base, (AVRational){1, 90000});
 
-    *surface = qf->surface;
+    *new_frame = qf;
 
     return 0;
 }
@@ -885,15 +905,21 @@ int ff_qsv_encode(AVCodecContext *avctx, QSVEncContext *q,
 
     mfxFrameSurface1 *surf = NULL;
     mfxSyncPoint sync      = NULL;
+    QSVFrame *qsv_frame = NULL;
+    mfxEncodeCtrl* enc_ctrl = NULL;
     int ret;
 
     if (frame) {
-        ret = submit_frame(q, frame, &surf);
+        ret = submit_frame(q, frame, &qsv_frame);
         if (ret < 0) {
             av_log(avctx, AV_LOG_ERROR, "Error submitting the frame for encoding.\n");
             return ret;
         }
     }
+    if (qsv_frame) {
+        surf = qsv_frame->surface;
+        enc_ctrl = &qsv_frame->enc_ctrl;
+    }
 
     ret = av_new_packet(&new_pkt, q->packet_size);
     if (ret < 0) {
@@ -909,8 +935,12 @@ int ff_qsv_encode(AVCodecContext *avctx, QSVEncContext *q,
     bs->Data      = new_pkt.data;
     bs->MaxLength = new_pkt.size;
 
+    if (q->set_encode_ctrl_cb) {
+        q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl);
+    }
+
     do {
-        ret = MFXVideoENCODE_EncodeFrameAsync(q->session, NULL, surf, bs, &sync);
+        ret = MFXVideoENCODE_EncodeFrameAsync(q->session, enc_ctrl, surf, bs, &sync);
         if (ret == MFX_WRN_DEVICE_BUSY) {
             av_usleep(500);
             continue;
@@ -1010,6 +1040,7 @@ int ff_qsv_enc_close(AVCodecContext *avctx, QSVEncContext *q)
     while (cur) {
         q->work_frames = cur->next;
         av_frame_free(&cur->frame);
+        av_free(cur->enc_ctrl.Payload);
         av_freep(&cur);
         cur = q->work_frames;
     }

libavcodec/qsvenc.h

@@ -69,6 +69,8 @@
 { "adaptive_i",     "Adaptive I-frame placement",             OFFSET(qsv.adaptive_i),     AV_OPT_TYPE_INT, { .i64 = -1 }, -1,          1, VE },                         \
 { "adaptive_b",     "Adaptive B-frame placement",             OFFSET(qsv.adaptive_b),     AV_OPT_TYPE_INT, { .i64 = -1 }, -1,          1, VE },                         \
 
+typedef int SetEncodeCtrlCB (AVCodecContext *avctx,
+                             const AVFrame *frame, mfxEncodeCtrl* enc_ctrl);
 typedef struct QSVEncContext {
     AVCodecContext *avctx;
 
@@ -131,7 +133,9 @@ typedef struct QSVEncContext {
     int int_ref_qp_delta;
     int recovery_point_sei;
 
+    int a53_cc;
     char *load_plugins;
+    SetEncodeCtrlCB *set_encode_ctrl_cb;
 } QSVEncContext;
 
 int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q);

libavcodec/qsvenc_h264.c

@@ -40,10 +40,75 @@ typedef struct QSVH264EncContext {
     QSVEncContext qsv;
 } QSVH264EncContext;
 
+static int qsv_h264_set_encode_ctrl(AVCodecContext *avctx,
+                                    const AVFrame *frame, mfxEncodeCtrl* enc_ctrl)
+{
+    AVFrameSideData *side_data = NULL;
+    QSVH264EncContext *qh264 = avctx->priv_data;
+    QSVEncContext *q = &qh264->qsv;
+
+    if (q->a53_cc && frame) {
+        side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC);
+        if (side_data) {
+
+            int sei_payload_size = 0;
+            mfxU8* sei_data = NULL;
+            mfxPayload* payload = NULL;
+
+            sei_payload_size = side_data->size + 13;
+
+            sei_data = av_mallocz(sei_payload_size);
+            if (!sei_data) {
+                av_log(avctx, AV_LOG_ERROR, "No memory for CC, skipping...\n");
+                return AVERROR(ENOMEM);
+            }
+
+            // SEI header
+            sei_data[0] = 4;
+            sei_data[1] = sei_payload_size - 2; // size of SEI data
+
+            // country code
+            sei_data[2] = 181;
+            sei_data[3] = 0;
+            sei_data[4] = 49;
+
+            // ATSC_identifier - using 'GA94' only
+            AV_WL32(sei_data + 5,
+                MKTAG('G', 'A', '9', '4'));
+            sei_data[9] = 3;
+            sei_data[10] =
+                ((side_data->size/3) & 0x1f) | 0xC0;
+
+            sei_data[11] = 0xFF; // reserved
+
+            memcpy(sei_data + 12, side_data->data, side_data->size);
+
+            sei_data[side_data->size+12] = 255;
+
+            payload = av_mallocz(sizeof(mfxPayload));
+            if (!payload) {
+                av_log(avctx, AV_LOG_ERROR, "No memory, skipping captions\n");
+                av_freep(&sei_data);
+                return AVERROR(ENOMEM);
+            }
+            payload->BufSize = side_data->size + 13;
+            payload->NumBit = payload->BufSize * 8;
+            payload->Type = 4;
+            payload->Data = sei_data;
+
+            enc_ctrl->NumExtParam = 0;
+            enc_ctrl->NumPayload = 1;
+            enc_ctrl->Payload[0] = payload;
+        }
+    }
+    return 0;
+}
+
 static av_cold int qsv_enc_init(AVCodecContext *avctx)
 {
     QSVH264EncContext *q = avctx->priv_data;
 
+    q->qsv.set_encode_ctrl_cb = qsv_h264_set_encode_ctrl;
     return ff_qsv_enc_init(avctx, &q->qsv);
 }
 
@@ -103,6 +168,7 @@ static const AVOption options[] = {
     { "main"    , NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_PROFILE_AVC_MAIN     }, INT_MIN, INT_MAX,     VE, "profile" },
     { "high"    , NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_PROFILE_AVC_HIGH     }, INT_MIN, INT_MAX,     VE, "profile" },
 
+    { "a53cc" , "Use A53 Closed Captions (if available)", OFFSET(qsv.a53_cc), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, VE},
     { NULL },
 };