virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2025-08-15 14:13:16 +02:00
Code Issues Releases Activity

avcodec/videotoolboxenc: fix undefined behavior with rc_max_rate=0

Browse Source 
On macOS, a zero rc_max_rate cause an error from
VTSessionSetProperty(kVTCompressionPropertyKey_DataRateLimits).

on iOS (depending on device/version), a zero rc_max_rate cause invalid
arguments from the vtenc_output_callback after few frames and then a crash
within the VideoToolbox library.

Signed-off-by: Aman Gupta <aman@tmm1.net>
This commit is contained in:
Thomas Guillem
2018-07-04 09:05:22 +02:00
committed by Aman Gupta
parent 37644f0477
commit 93e157f40f
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
libavcodec/videotoolboxenc.c
View File

@@ -1017,7 +1017,7 @@ static int vtenc_create_encoder(AVCodecContext *avctx,
return AVERROR_EXTERNAL; return AVERROR_EXTERNAL;
} }
if (vtctx->codec_id == AV_CODEC_ID_H264) { if (vtctx->codec_id == AV_CODEC_ID_H264 && max_rate > 0) {
// kVTCompressionPropertyKey_DataRateLimits is not available for HEVC // kVTCompressionPropertyKey_DataRateLimits is not available for HEVC
bytes_per_second_value = max_rate >> 3; bytes_per_second_value = max_rate >> 3;
bytes_per_second = CFNumberCreate(kCFAllocatorDefault, bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
@@ -1058,7 +1058,10 @@ static int vtenc_create_encoder(AVCodecContext *avctx,
av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status); av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
return AVERROR_EXTERNAL; return AVERROR_EXTERNAL;
} }
}
if (vtctx->codec_id == AV_CODEC_ID_H264) {
// kVTCompressionPropertyKey_ProfileLevel is not available for HEVC
if (profile_level) { if (profile_level) {
status = VTSessionSetProperty(vtctx->session, status = VTSessionSetProperty(vtctx->session,
kVTCompressionPropertyKey_ProfileLevel, kVTCompressionPropertyKey_ProfileLevel,