From a41a2efc85f8c88caec10040ee437562f9d0b947 Mon Sep 17 00:00:00 2001 From: rcombs Date: Sat, 13 Nov 2021 02:43:06 -0600 Subject: [PATCH] lavc/videotoolbox: add VP9 hardware acceleration On M1 Max, this supports profiles 0 and 2, but not 1 and 3. --- Changelog | 1 + configure | 4 + libavcodec/Makefile | 1 + libavcodec/hwaccels.h | 1 + libavcodec/videotoolbox.c | 17 ++++- libavcodec/videotoolbox_vp9.c | 140 ++++++++++++++++++++++++++++++++++ libavcodec/vp9.c | 9 ++- libavcodec/vt_internal.h | 1 + 8 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 libavcodec/videotoolbox_vp9.c diff --git a/Changelog b/Changelog index a66231b4d8..a520b832b2 100644 --- a/Changelog +++ b/Changelog @@ -33,6 +33,7 @@ version : - colorspectrum source video filter - RTP packetizer for uncompressed video (RFC 4175) - bitpacked encoder +- VideoToolbox VP9 hwaccel version 4.4: diff --git a/configure b/configure index d7f9429ecd..07b168af72 100755 --- a/configure +++ b/configure @@ -2337,6 +2337,7 @@ TOOLCHAIN_FEATURES=" TYPES_LIST=" kCMVideoCodecType_HEVC kCMVideoCodecType_HEVCWithAlpha + kCMVideoCodecType_VP9 kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange @@ -3092,6 +3093,8 @@ vp9_vaapi_hwaccel_deps="vaapi VADecPictureParameterBufferVP9_bit_depth" vp9_vaapi_hwaccel_select="vp9_decoder" vp9_vdpau_hwaccel_deps="vdpau VdpPictureInfoVP9" vp9_vdpau_hwaccel_select="vp9_decoder" +vp9_videotoolbox_hwaccel_deps="videotoolbox" +vp9_videotoolbox_hwaccel_select="vp9_decoder" wmv3_d3d11va_hwaccel_select="vc1_d3d11va_hwaccel" wmv3_d3d11va2_hwaccel_select="vc1_d3d11va2_hwaccel" wmv3_dxva2_hwaccel_select="vc1_dxva2_hwaccel" @@ -6296,6 +6299,7 @@ enabled videotoolbox && { check_lib coreservices CoreServices/CoreServices.h UTGetOSTypeFromString "-framework CoreServices" check_func_headers CoreMedia/CMFormatDescription.h kCMVideoCodecType_HEVC "-framework CoreMedia" check_func_headers CoreMedia/CMFormatDescription.h kCMVideoCodecType_HEVCWithAlpha "-framework CoreMedia" + check_func_headers CoreMedia/CMFormatDescription.h kCMVideoCodecType_VP9 "-framework CoreMedia" check_func_headers CoreVideo/CVPixelBuffer.h kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange "-framework CoreVideo" check_func_headers CoreVideo/CVPixelBuffer.h kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange "-framework CoreVideo" check_func_headers CoreVideo/CVPixelBuffer.h kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange "-framework CoreVideo" diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 6be90cd5c0..4122a9b144 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -976,6 +976,7 @@ OBJS-$(CONFIG_VP9_DXVA2_HWACCEL) += dxva2_vp9.o OBJS-$(CONFIG_VP9_NVDEC_HWACCEL) += nvdec_vp9.o OBJS-$(CONFIG_VP9_VAAPI_HWACCEL) += vaapi_vp9.o OBJS-$(CONFIG_VP9_VDPAU_HWACCEL) += vdpau_vp9.o +OBJS-$(CONFIG_VP9_VIDEOTOOLBOX_HWACCEL) += videotoolbox_vp9.o OBJS-$(CONFIG_VP8_QSV_HWACCEL) += qsvdec.o # libavformat dependencies diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h index 8e54cf73f9..65e778f3e4 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h @@ -74,6 +74,7 @@ extern const AVHWAccel ff_vp9_dxva2_hwaccel; extern const AVHWAccel ff_vp9_nvdec_hwaccel; extern const AVHWAccel ff_vp9_vaapi_hwaccel; extern const AVHWAccel ff_vp9_vdpau_hwaccel; +extern const AVHWAccel ff_vp9_videotoolbox_hwaccel; extern const AVHWAccel ff_wmv3_d3d11va_hwaccel; extern const AVHWAccel ff_wmv3_d3d11va2_hwaccel; extern const AVHWAccel ff_wmv3_dxva2_hwaccel; diff --git a/libavcodec/videotoolbox.c b/libavcodec/videotoolbox.c index a11d494915..5f08488a6b 100644 --- a/libavcodec/videotoolbox.c +++ b/libavcodec/videotoolbox.c @@ -33,6 +33,7 @@ #include "hevcdec.h" #include "mpegvideo.h" #include +#include #include #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder @@ -46,6 +47,10 @@ enum { kCMVideoCodecType_HEVC = 'hvc1' }; #endif +#if !HAVE_KCMVIDEOCODECTYPE_VP9 +enum { kCMVideoCodecType_VP9 = 'vp09' }; +#endif + #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12 typedef struct VTHWFrame { @@ -816,6 +821,11 @@ static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec if (data) CFDictionarySetValue(avc_info, CFSTR("hvcC"), data); break; + case kCMVideoCodecType_VP9 : + data = ff_videotoolbox_vpcc_extradata_create(avctx); + if (data) + CFDictionarySetValue(avc_info, CFSTR("vpcC"), data); + break; default: break; } @@ -863,12 +873,15 @@ static int videotoolbox_start(AVCodecContext *avctx) case AV_CODEC_ID_MPEG4 : videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG4Video; break; + case AV_CODEC_ID_VP9 : + videotoolbox->cm_codec_type = kCMVideoCodecType_VP9; + break; default : break; } -#ifdef __MAC_10_11 - if (__builtin_available(macOS 10.11, *)) { +#if defined(MAC_OS_VERSION_11_0) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0) + if (__builtin_available(macOS 11.0, *)) { VTRegisterSupplementalVideoDecoderIfAvailable(videotoolbox->cm_codec_type); } #endif diff --git a/libavcodec/videotoolbox_vp9.c b/libavcodec/videotoolbox_vp9.c new file mode 100644 index 0000000000..1b6d08f00b --- /dev/null +++ b/libavcodec/videotoolbox_vp9.c @@ -0,0 +1,140 @@ +/* + * Videotoolbox hardware acceleration for VP9 + * + * copyright (c) 2021 rcombs + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" +#include "videotoolbox.h" +#include "libavutil/hwcontext_videotoolbox.h" +#include "vt_internal.h" +#include "libavutil/avutil.h" +#include "libavutil/frame.h" +#include "libavutil/hwcontext.h" +#include "libavutil/intreadwrite.h" +#include "libavutil/pixdesc.h" +#include "decode.h" +#include "internal.h" +#include "vp9shared.h" + +enum VPX_CHROMA_SUBSAMPLING +{ + VPX_SUBSAMPLING_420_VERTICAL = 0, + VPX_SUBSAMPLING_420_COLLOCATED_WITH_LUMA = 1, + VPX_SUBSAMPLING_422 = 2, + VPX_SUBSAMPLING_444 = 3, +}; + +static int get_vpx_chroma_subsampling(enum AVPixelFormat pixel_format, + enum AVChromaLocation chroma_location) +{ + int chroma_w, chroma_h; + if (av_pix_fmt_get_chroma_sub_sample(pixel_format, &chroma_w, &chroma_h) == 0) { + if (chroma_w == 1 && chroma_h == 1) { + return (chroma_location == AVCHROMA_LOC_LEFT) + ? VPX_SUBSAMPLING_420_VERTICAL + : VPX_SUBSAMPLING_420_COLLOCATED_WITH_LUMA; + } else if (chroma_w == 1 && chroma_h == 0) { + return VPX_SUBSAMPLING_422; + } else if (chroma_w == 0 && chroma_h == 0) { + return VPX_SUBSAMPLING_444; + } + } + return -1; +} + +CFDataRef ff_videotoolbox_vpcc_extradata_create(AVCodecContext *avctx) +{ + const VP9SharedContext *h = avctx->priv_data; + CFDataRef data = NULL; + uint8_t *p; + int vt_extradata_size; + uint8_t *vt_extradata; + int subsampling = get_vpx_chroma_subsampling(avctx->sw_pix_fmt, avctx->chroma_sample_location); + + vt_extradata_size = 1 + 3 + 6 + 2; + vt_extradata = av_malloc(vt_extradata_size); + + if (subsampling < 0) + return NULL; + + if (!vt_extradata) + return NULL; + + p = vt_extradata; + + *p++ = 1; /* version */ + AV_WB24(p + 1, 0); /* flags */ + p += 3; + + *p++ = h->h.profile; + *p++ = avctx->level; + *p++ = (h->h.bpp << 4) | (subsampling << 1) | (avctx->color_range == AVCOL_RANGE_JPEG); + *p++ = avctx->color_primaries; + *p++ = avctx->color_trc; + *p++ = avctx->colorspace; + + AV_WB16(p + 0, 0); + p += 2; + + av_assert0(p - vt_extradata == vt_extradata_size); + + data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size); + av_free(vt_extradata); + return data; +} + +static int videotoolbox_vp9_start_frame(AVCodecContext *avctx, + const uint8_t *buffer, + uint32_t size) +{ + return 0; +} + +static int videotoolbox_vp9_decode_slice(AVCodecContext *avctx, + const uint8_t *buffer, + uint32_t size) +{ + VTContext *vtctx = avctx->internal->hwaccel_priv_data; + + return ff_videotoolbox_buffer_copy(vtctx, buffer, size); +} + +static int videotoolbox_vp9_end_frame(AVCodecContext *avctx) +{ + const VP9SharedContext *h = avctx->priv_data; + AVFrame *frame = h->frames[CUR_FRAME].tf.f; + + return ff_videotoolbox_common_end_frame(avctx, frame); +} + +const AVHWAccel ff_vp9_videotoolbox_hwaccel = { + .name = "vp9_videotoolbox", + .type = AVMEDIA_TYPE_VIDEO, + .id = AV_CODEC_ID_VP9, + .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX, + .alloc_frame = ff_videotoolbox_alloc_frame, + .start_frame = videotoolbox_vp9_start_frame, + .decode_slice = videotoolbox_vp9_decode_slice, + .end_frame = videotoolbox_vp9_end_frame, + .frame_params = ff_videotoolbox_frame_params, + .init = ff_videotoolbox_common_init, + .uninit = ff_videotoolbox_uninit, + .priv_data_size = sizeof(VTContext), +}; diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c index 8317ac6bd8..24841c5b9f 100644 --- a/libavcodec/vp9.c +++ b/libavcodec/vp9.c @@ -181,7 +181,8 @@ static int update_size(AVCodecContext *avctx, int w, int h) CONFIG_VP9_D3D11VA_HWACCEL * 2 + \ CONFIG_VP9_NVDEC_HWACCEL + \ CONFIG_VP9_VAAPI_HWACCEL + \ - CONFIG_VP9_VDPAU_HWACCEL) + CONFIG_VP9_VDPAU_HWACCEL + \ + CONFIG_VP9_VIDEOTOOLBOX_HWACCEL) enum AVPixelFormat pix_fmts[HWACCEL_MAX + 2], *fmtp = pix_fmts; VP9Context *s = avctx->priv_data; uint8_t *p; @@ -212,6 +213,9 @@ static int update_size(AVCodecContext *avctx, int w, int h) #endif #if CONFIG_VP9_VDPAU_HWACCEL *fmtp++ = AV_PIX_FMT_VDPAU; +#endif +#if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL + *fmtp++ = AV_PIX_FMT_VIDEOTOOLBOX; #endif break; case AV_PIX_FMT_YUV420P12: @@ -1893,6 +1897,9 @@ const AVCodec ff_vp9_decoder = { #endif #if CONFIG_VP9_VDPAU_HWACCEL HWACCEL_VDPAU(vp9), +#endif +#if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL + HWACCEL_VIDEOTOOLBOX(vp9), #endif NULL }, diff --git a/libavcodec/vt_internal.h b/libavcodec/vt_internal.h index e2f279d0fc..54a11fd1b5 100644 --- a/libavcodec/vt_internal.h +++ b/libavcodec/vt_internal.h @@ -64,5 +64,6 @@ int ff_videotoolbox_h264_decode_slice(AVCodecContext *avctx, int ff_videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame); CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx); CFDataRef ff_videotoolbox_hvcc_extradata_create(AVCodecContext *avctx); +CFDataRef ff_videotoolbox_vpcc_extradata_create(AVCodecContext *avctx); #endif /* AVCODEC_VT_INTERNAL_H */