avcodec/nvenc: add MV-HEVC encoding support

Added support for MV-HEVC encoding for stereoscopic videos (2 views
only). Compatible with the framepack filter when using the
AV_STEREO3D_FRAMESEQUENCE format.

Signed-off-by: Diego de Souza <ddesouza@nvidia.com>

libavcodec/nvenc.c

@@ -37,6 +37,8 @@
 #include "libavutil/timecode_internal.h"
 #include "libavutil/mathematics.h"
 #include "libavutil/mastering_display_metadata.h"
+#include "libavutil/stereo3d.h"
+#include "libavutil/tdrdi.h"
 #include "atsc_a53.h"
 #include "codec_desc.h"
 #include "encode.h"
@@ -660,6 +662,14 @@ static int nvenc_check_capabilities(AVCodecContext *avctx)
 
     ctx->support_dyn_bitrate = nvenc_check_cap(avctx, NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE);
 
+#ifdef NVENC_HAVE_MVHEVC
+    ctx->multiview_supported = nvenc_check_cap(avctx, NV_ENC_CAPS_SUPPORT_MVHEVC_ENCODE) > 0;
+    if(ctx->profile == NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN && !ctx->multiview_supported) {
+        av_log(avctx, AV_LOG_WARNING, "Multiview not supported by the device\n");
+        return AVERROR(ENOSYS);
+    }
+#endif
+
     return 0;
 }
 
@@ -1518,6 +1528,26 @@ static av_cold int nvenc_setup_hevc_config(AVCodecContext *avctx)
 
     hevc->outputPictureTimingSEI = 1;
 
+#ifdef NVENC_HAVE_MVHEVC
+    if (ctx->multiview_supported && (ctx->profile == NV_ENC_HEVC_PROFILE_MAIN || ctx->profile == NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN)) {
+        const AVFrameSideData *sd_stereo3d = av_frame_side_data_get(avctx->decoded_side_data, avctx->nb_decoded_side_data, AV_FRAME_DATA_STEREO3D);
+        const AVFrameSideData *sd_tdrdi = av_frame_side_data_get(avctx->decoded_side_data, avctx->nb_decoded_side_data, AV_FRAME_DATA_3D_REFERENCE_DISPLAYS);
+        const AVStereo3D *stereo3d = sd_stereo3d ? (const AVStereo3D*)sd_stereo3d->data : NULL;
+
+        if (sd_tdrdi && stereo3d && stereo3d->type == AV_STEREO3D_FRAMESEQUENCE)
+            ctx->profile = NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN;
+
+        if (ctx->profile == NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN && stereo3d &&
+            stereo3d->type != AV_STEREO3D_2D &&
+            stereo3d->type != AV_STEREO3D_UNSPEC &&
+            stereo3d->type != AV_STEREO3D_FRAMESEQUENCE)
+        {
+            av_log(avctx, AV_LOG_WARNING, "Unsupported multiview input, disabling multiview encoding.\n");
+            ctx->profile = NV_ENC_HEVC_PROFILE_MAIN;
+        }
+    }
+#endif
+
     switch (ctx->profile) {
     case NV_ENC_HEVC_PROFILE_MAIN:
         cc->profileGUID = NV_ENC_HEVC_PROFILE_MAIN_GUID;
@@ -1531,6 +1561,18 @@ static av_cold int nvenc_setup_hevc_config(AVCodecContext *avctx)
         cc->profileGUID = NV_ENC_HEVC_PROFILE_FREXT_GUID;
         avctx->profile  = AV_PROFILE_HEVC_REXT;
         break;
+#ifdef NVENC_HAVE_MVHEVC
+    case NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN:
+        cc->profileGUID = NV_ENC_HEVC_PROFILE_MAIN_GUID;
+        avctx->profile  = AV_PROFILE_HEVC_MULTIVIEW_MAIN;
+        ctx->multiview  = 1;
+
+        hevc->enableMVHEVC = 1;
+        hevc->outputHevc3DReferenceDisplayInfo = 1;
+
+        av_log(avctx, AV_LOG_VERBOSE, "Enabling MV HEVC encoding.\n");
+        break;
+#endif
     }
 
     // force setting profile as main10 if input is 10 bit or if it should be encoded as 10 bit
@@ -1545,6 +1587,13 @@ static av_cold int nvenc_setup_hevc_config(AVCodecContext *avctx)
         avctx->profile = AV_PROFILE_HEVC_REXT;
     }
 
+#ifdef NVENC_HAVE_MVHEVC
+    if (ctx->multiview && avctx->profile != AV_PROFILE_HEVC_MULTIVIEW_MAIN) {
+        av_log(avctx, AV_LOG_ERROR, "Multiview encoding only works for Main profile content.\n");
+        return AVERROR(EINVAL);
+    }
+#endif
+
     hevc->chromaFormatIDC = IS_YUV444(ctx->data_pix_fmt) ? 3 : IS_YUV422(ctx->data_pix_fmt) ? 2 : 1;
 
 #ifdef NVENC_HAVE_NEW_BIT_DEPTH_API
@@ -2565,6 +2614,9 @@ static int nvenc_set_timestamp(AVCodecContext *avctx,
 
     // This can be more than necessary, but we don't know the real reorder delay.
     delay = FFMAX(ctx->encode_config.frameIntervalP - 1, 0);
+#ifdef NVENC_HAVE_MVHEVC
+    delay *= ctx->multiview ? 2 : 1;
+#endif
     if (ctx->output_frame_num >= delay) {
         pkt->dts = timestamp_queue_dequeue(ctx->timestamp_list);
         ctx->output_frame_num++;
@@ -3047,6 +3099,9 @@ static int nvenc_send_frame(AVCodecContext *avctx, const AVFrame *frame)
     MASTERING_DISPLAY_INFO mastering_disp_info = { 0 };
     CONTENT_LIGHT_LEVEL content_light_level = { 0 };
 #endif
+#ifdef NVENC_HAVE_MVHEVC
+    HEVC_3D_REFERENCE_DISPLAY_INFO ref_disp_info = { 0 };
+#endif
 
     NvencContext *ctx = avctx->priv_data;
     NvencDynLoadFunctions *dl_fn = &ctx->nvenc_dload_funcs;
@@ -3117,6 +3172,53 @@ static int nvenc_send_frame(AVCodecContext *avctx, const AVFrame *frame)
             return res;
 #endif
 
+#ifdef NVENC_HAVE_MVHEVC
+        if (ctx->multiview) {
+            const AVFrameSideData *sd_tdrdi = av_frame_get_side_data(frame, AV_FRAME_DATA_3D_REFERENCE_DISPLAYS);
+            const AVFrameSideData *sd_view_id = av_frame_get_side_data(frame, AV_FRAME_DATA_VIEW_ID);
+
+            if (sd_view_id)
+                ctx->next_view_id = *(int*)sd_view_id->data;
+
+            pic_params.codecPicParams.hevcPicParams.viewId = ctx->next_view_id;
+
+            if (sd_tdrdi) {
+                AV3DReferenceDisplaysInfo *tdrdi = (AV3DReferenceDisplaysInfo*)sd_tdrdi->data;
+
+                ref_disp_info.refViewingDistanceFlag = tdrdi->ref_viewing_distance_flag;
+                ref_disp_info.precRefViewingDist = tdrdi->prec_ref_viewing_dist;
+                ref_disp_info.precRefDisplayWidth = tdrdi->prec_ref_display_width;
+
+                ref_disp_info.numRefDisplaysMinus1 = tdrdi->num_ref_displays - 1;
+
+                for (i = 0; i < tdrdi->num_ref_displays &&
+                            i < FF_ARRAY_ELEMS(ref_disp_info.leftViewId); i++) {
+                    const AV3DReferenceDisplay *display = av_tdrdi_get_display(tdrdi, i);
+                    ref_disp_info.leftViewId[i] = display->left_view_id;
+                    ref_disp_info.rightViewId[i] = display->right_view_id;
+                    ref_disp_info.exponentRefDisplayWidth[i] = display->exponent_ref_display_width;
+                    ref_disp_info.mantissaRefDisplayWidth[i] = display->mantissa_ref_display_width;
+                    ref_disp_info.exponentRefViewingDistance[i] = display->exponent_ref_viewing_distance;
+                    ref_disp_info.mantissaRefViewingDistance[i] = display->mantissa_ref_viewing_distance;
+                    ref_disp_info.additionalShiftPresentFlag[i] = display->additional_shift_present_flag;
+                    ref_disp_info.numSampleShiftPlus512[i] = display->num_sample_shift + 512;
+                }
+
+                pic_params.codecPicParams.hevcPicParams.p3DReferenceDisplayInfo = &ref_disp_info;
+                ctx->display_sei_sent = 1;
+            } else if (!ctx->display_sei_sent) {
+                ref_disp_info.precRefDisplayWidth = 31;
+                ref_disp_info.leftViewId[0] = 0;
+                ref_disp_info.rightViewId[0] = 1;
+
+                pic_params.codecPicParams.hevcPicParams.p3DReferenceDisplayInfo = &ref_disp_info;
+                ctx->display_sei_sent = 1;
+            }
+
+            ctx->next_view_id = !ctx->next_view_id;
+        }
+#endif
+
         res = nvenc_store_frame_data(avctx, &pic_params, frame);
         if (res < 0)
             return res;

libavcodec/nvenc.h

@@ -107,6 +107,7 @@ typedef void ID3D11Device;
 #define NVENC_HAVE_AV1_UHQ_TUNING
 #define NVENC_HAVE_H264_AND_AV1_TEMPORAL_FILTER
 #define NVENC_HAVE_HEVC_AND_AV1_MASTERING_METADATA
+#define NVENC_HAVE_MVHEVC
 #endif
 
 typedef struct NvencSurface
@@ -180,6 +181,11 @@ enum {
     NV_ENC_HEVC_PROFILE_MAIN,
     NV_ENC_HEVC_PROFILE_MAIN_10,
     NV_ENC_HEVC_PROFILE_REXT,
+#ifdef NVENC_HAVE_MVHEVC
+    NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN,
+#endif
+
+    NV_ENC_HEVC_PROFILE_COUNT
 };
 
 enum {
@@ -253,6 +259,7 @@ typedef struct NvencContext
     void *nvencoder;
 
     uint32_t frame_idx_counter;
+    uint32_t next_view_id;
 
     int preset;
     int profile;
@@ -311,6 +318,8 @@ typedef struct NvencContext
     int split_encode_mode;
     int mdm, cll;
     int cbr_padding;
+    int multiview, multiview_supported;
+    int display_sei_sent;
 } NvencContext;
 
 int ff_nvenc_encode_init(AVCodecContext *avctx);

libavcodec/nvenc_hevc.c

@@ -60,10 +60,13 @@ static const AVOption options[] = {
     { "ull",         "Ultra low latency",                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY },        0, 0, VE, .unit = "tune" },
     { "lossless",    "Lossless",                            0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_TUNING_INFO_LOSSLESS },                 0, 0, VE, .unit = "tune" },
 #endif
-    { "profile",      "Set the encoding profile",           OFFSET(profile),      AV_OPT_TYPE_INT,   { .i64 = NV_ENC_HEVC_PROFILE_MAIN }, NV_ENC_HEVC_PROFILE_MAIN, AV_PROFILE_HEVC_REXT, VE, .unit = "profile" },
+    { "profile",      "Set the encoding profile",           OFFSET(profile),      AV_OPT_TYPE_INT,   { .i64 = NV_ENC_HEVC_PROFILE_MAIN }, NV_ENC_HEVC_PROFILE_MAIN, NV_ENC_HEVC_PROFILE_COUNT - 1, VE, .unit = "profile" },
     { "main",         "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_HEVC_PROFILE_MAIN },    0, 0, VE, .unit = "profile" },
     { "main10",       "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_HEVC_PROFILE_MAIN_10 }, 0, 0, VE, .unit = "profile" },
     { "rext",         "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_HEVC_PROFILE_REXT },    0, 0, VE, .unit = "profile" },
+#ifdef NVENC_HAVE_MVHEVC
+    { "mv",           "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_HEVC_PROFILE_MULTIVIEW_MAIN }, 0, 0, VE, .unit = "profile" },
+#endif
     { "level",        "Set the encoding level restriction", OFFSET(level),        AV_OPT_TYPE_INT,   { .i64 = NV_ENC_LEVEL_AUTOSELECT }, NV_ENC_LEVEL_AUTOSELECT, NV_ENC_LEVEL_HEVC_62, VE, .unit = "level" },
     { "auto",         "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_LEVEL_AUTOSELECT },  0, 0, VE,  .unit = "level" },
     { "1",            "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = NV_ENC_LEVEL_HEVC_1 },      0, 0, VE,  .unit = "level" },

libavcodec/version.h

@@ -30,7 +30,7 @@
 #include "version_major.h"
 
 #define LIBAVCODEC_VERSION_MINOR   4
-#define LIBAVCODEC_VERSION_MICRO 102
+#define LIBAVCODEC_VERSION_MICRO 103
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \